Skip to main content

runnerd_core/
store.rs

1//! The job history: a SQLite database of jobs, their runners, steps, samples and logs, the events clients resume
2//! from, cache sizes, the paused repositories and where the notifier got to. Owns the schema, the one writer and
3//! the event broadcast. Writes are messages to a writer thread, so the engine never waits on the disk; reads and
4//! pruning run on Tokio's blocking pool, pruning on its own connection, a batch at a time. It decides nothing about
5//! scheduling and stores no secret: no token, no JIT config, no runner `_diag` log.
6
7use std::collections::{BTreeMap, BTreeSet};
8use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt};
9use std::path::{Path, PathBuf};
10use std::sync::atomic::{AtomicBool, Ordering};
11use std::sync::{Arc, Mutex, TryLockError};
12use std::time::Duration;
13
14use rusqlite::{Connection, OpenFlags};
15use tokio::sync::{OwnedSemaphorePermit, Semaphore, broadcast, mpsc, oneshot};
16use tracing::warn;
17
18use crate::admission::trace::Trace;
19use crate::api::{self, Event, EventEnvelope};
20use crate::config::History;
21use crate::github::{JobDetails, QueuedJob};
22use crate::observe::metrics::{CpuHeld, Events};
23use crate::scheduler::warm;
24use crate::sizing;
25use crate::sync::lock;
26
27mod prune;
28mod read;
29mod write;
30
31/// The schema a new database is created with, at [`VERSION`].
32const SCHEMA: &str = include_str!("store/schema.sql");
33
34/// The oldest schema version a database is brought up from: 0.0.19's (ADR 0073).
35const OLDEST: u32 = 15;
36
37/// The steps from [`OLDEST`] on, in order, each with the version it brings a database to. A change to the schema
38/// adds one here and makes the same change to `schema.sql`, which a new database gets instead.
39const MIGRATIONS: &[(u32, &str)] = &[
40    (
41        16,
42        "ALTER TABLE jobs ADD COLUMN unseen INTEGER NOT NULL DEFAULT 0;",
43    ),
44    (
45        17,
46        "ALTER TABLE jobs ADD COLUMN cpu_periods INTEGER;\n\
47         ALTER TABLE jobs ADD COLUMN cpu_throttled INTEGER;\n\
48         ALTER TABLE jobs ADD COLUMN cpu_stalled INTEGER;\n\
49         ALTER TABLE samples ADD COLUMN cpu_periods INTEGER;\n\
50         ALTER TABLE samples ADD COLUMN cpu_throttled INTEGER;\n\
51         ALTER TABLE samples ADD COLUMN cpu_stalled INTEGER;",
52    ),
53    (
54        18,
55        "ALTER TABLE jobs ADD COLUMN slot_waited INTEGER NOT NULL DEFAULT 0;",
56    ),
57    (
58        19,
59        "ALTER TABLE jobs ADD COLUMN seen_at INTEGER;\n\
60         ALTER TABLE runners ADD COLUMN purpose TEXT;\n\
61         ALTER TABLE runners ADD COLUMN stopped_for TEXT;",
62    ),
63    (
64        20,
65        "ALTER TABLE jobs ADD COLUMN runner_fault INTEGER NOT NULL DEFAULT 0;",
66    ),
67    (21, "ALTER TABLE runners ADD COLUMN cancel_by INTEGER;"),
68];
69
70/// The schema version this build reads and writes, the database's `user_version`: its last migration's.
71const VERSION: u32 = match MIGRATIONS.last() {
72    Some((version, _)) => *version,
73    None => OLDEST,
74};
75
76/// Bytes of log lines sent and not yet stored, at most: a job's capture waits beyond it, so a noisy job or a slow
77/// disk holds back that job's log, not the daemon's memory.
78pub(crate) const LOG_BACKLOG: u64 = 8 << 20;
79
80/// Host samples the writer holds before it stores them in one transaction: a minute's worth. One transaction per
81/// sample wrote about 300 MB a day to an idle daemon's WAL. A crash loses the samples held.
82const HOST_SAMPLE_BATCH: usize = 30;
83
84/// How many events a slow client may fall behind before its stream ends; it reconnects with `Last-Event-ID`.
85const EVENT_BUFFER: usize = 4096;
86
87/// Why the database cannot be used.
88#[derive(Debug, thiserror::Error)]
89pub enum StoreError {
90    /// SQLite failed.
91    #[error("{0}")]
92    Sqlite(#[from] rusqlite::Error),
93    /// A newer runnerd wrote the database.
94    #[error(
95        "the job history database is version {found}, newer than the {supported} this runnerd reads; \
96         run a newer runnerd or move the database aside"
97    )]
98    Newer {
99        /// The database's version.
100        found: u32,
101        /// The newest version this build knows.
102        supported: u32,
103    },
104    /// A runnerd before 0.0.19 wrote the database.
105    #[error(
106        "the job history database is version {found}, from a runnerd before 0.0.19; this runnerd reads versions \
107         from {oldest} on: move the database aside"
108    )]
109    Older {
110        /// The database's version.
111        found: u32,
112        /// The oldest version this build reads.
113        oldest: u32,
114    },
115    /// The directory or file cannot be created.
116    #[error("{0}")]
117    Io(#[from] std::io::Error),
118    /// A blocking read did not finish.
119    #[error("a database read stopped: {0}")]
120    Join(#[from] tokio::task::JoinError),
121    /// The database's `-wal` file holds changes not yet in it, and this user cannot open the database the way that
122    /// reads them.
123    #[error(
124        "{} holds changes not yet in the job history, which this user cannot read; run runnerd doctor as the user \
125         the daemon runs as",
126        .0.display()
127    )]
128    Unread(PathBuf),
129}
130
131impl StoreError {
132    /// Whether a read stopped because the runtime shuts down, as when the daemon exits, rather than because the
133    /// history failed: the runtime cancels the blocking reads it has not run.
134    pub fn is_shutdown(&self) -> bool {
135        matches!(self, StoreError::Join(join) if join.is_cancelled())
136    }
137}
138
139/// A runner container as the history records it.
140#[derive(Debug, Clone, PartialEq, Eq)]
141pub struct RunnerRecord {
142    /// The runner's name, also its container's.
143    pub name: String,
144    /// `owner/name`.
145    pub repository: String,
146    /// The profile it runs in.
147    pub profile: String,
148    /// 1-based.
149    pub slot: usize,
150    /// The Docker container ID.
151    pub container: String,
152    /// When the container started, in milliseconds since the epoch.
153    pub started_at: u64,
154    /// The container's memory cap in bytes.
155    pub memory_cap: Option<u64>,
156    /// The container's CPU limit in millicores.
157    pub cpu_cap: Option<u64>,
158    /// How it was admitted; `None` for one a restart adopted without a record of it.
159    pub admission: Option<api::Admission>,
160    /// Why it started: `job`, `warm` or `burst`; `None` for one a restart adopted.
161    pub purpose: Option<&'static str>,
162}
163
164/// One sample of one runner's container.
165#[derive(Debug, Clone, PartialEq, Eq)]
166pub struct SampleRecord {
167    /// The runner.
168    pub runner: String,
169    /// When, in milliseconds since the epoch.
170    pub at: u64,
171    /// Millicores over the interval before it.
172    pub cpu: u64,
173    /// CPU time so far, in microseconds.
174    pub cpu_usage: u64,
175    /// Bytes without page cache (`observe::metrics::Usage::memory`).
176    pub memory: u64,
177    /// Bytes received since the container started.
178    pub network_received: u64,
179    /// Bytes sent since the container started.
180    pub network_sent: u64,
181    /// The container's `memory.events` so far.
182    pub events: Events,
183    /// What held the container's CPU back so far.
184    pub cpu_held: CpuHeld,
185}
186
187/// The last measurement of the cache volumes.
188#[derive(Debug, Clone, Default, PartialEq, Eq)]
189pub struct Measured {
190    /// When it was taken (seconds); none before the first measurement.
191    pub at: Option<u64>,
192    /// Each volume's size in bytes, by name.
193    pub bytes: BTreeMap<String, u64>,
194}
195
196/// The profiles and repositories as the stored events last announced them: what clients resuming after those
197/// events have.
198#[derive(Debug, Clone, Default, PartialEq, Eq)]
199pub struct Announced {
200    /// The last `profile.changed` of each profile not removed since, by name.
201    pub profiles: Vec<api::ProfileStatus>,
202    /// The last `repository.changed` of each repository not removed since, by name.
203    pub repositories: Vec<api::RepositoryStatus>,
204}
205
206/// A job the history has not closed, as the engine needs it after a restart.
207#[derive(Debug, Clone, PartialEq, Eq)]
208pub struct OpenJob {
209    /// GitHub's job ID.
210    pub id: u64,
211    /// `owner/name`.
212    pub repository: String,
213    /// `Queued` or `Running`.
214    pub state: api::JobState,
215    /// Its runner, once one took it.
216    pub runner: Option<String>,
217    /// When its runner was removed, in milliseconds since the epoch.
218    pub runner_removed_at: Option<u64>,
219    /// The workflow run it belongs to.
220    pub run_id: u64,
221    /// 1 for the run's first attempt.
222    pub run_attempt: u32,
223    /// The workflow's name, when known.
224    pub workflow: Option<String>,
225    /// When GitHub queued it, in milliseconds since the epoch.
226    pub queued_at: u64,
227    /// The labels it asked for, which tell its repository's pool.
228    pub labels: Vec<String>,
229}
230
231/// The job a slot ran last, as the history has it.
232#[derive(Debug, Clone, PartialEq, Eq)]
233pub struct SlotJob {
234    /// The profile the slot is in.
235    pub profile: String,
236    /// 1-based.
237    pub slot: usize,
238    /// The job, as admission tells jobs apart.
239    pub key: crate::admission::Key,
240}
241
242/// Where a job's log capture continues: the runner's page file and the byte offset in it.
243#[derive(Debug, Clone, Copy, PartialEq, Eq)]
244pub struct LogPosition {
245    /// The page number, from 1.
246    pub page: u64,
247    /// Bytes of that page already stored as lines.
248    pub offset: u64,
249}
250
251/// How far the mirror's log was counted: the time of the last response counted, in nanoseconds since the epoch as
252/// Docker logs it, and how many responses logged at exactly that time were counted, so the next read counts the
253/// ones it was too early to see.
254#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
255pub struct MirrorPosition {
256    /// Nanoseconds since the epoch.
257    pub at: u64,
258    /// Responses logged at `at` already counted.
259    pub seen: u64,
260}
261
262/// What the writer thread is asked to do, in order.
263#[derive(Debug)]
264enum Write {
265    Queued {
266        repository: String,
267        job: QueuedJob,
268        seen_at: u64,
269    },
270    RunnerStarted(RunnerRecord),
271    RunnerAdmission {
272        runner: String,
273        admission: api::Admission,
274    },
275    RunnerOomKilled(String),
276    RunnerStoppedFor {
277        runner: String,
278        reason: &'static str,
279    },
280    RunnerCancelBy {
281        runner: String,
282        at: u64,
283    },
284    HostWaited {
285        jobs: Vec<u64>,
286        ms: u64,
287    },
288    SlotWaited {
289        jobs: Vec<u64>,
290        ms: u64,
291    },
292    BuildCache {
293        runner: String,
294        counts: api::BuildCacheCounts,
295    },
296    RunnerRemoved {
297        runner: String,
298        at: u64,
299    },
300    RunnersGone {
301        running: Vec<String>,
302        at: u64,
303    },
304    Claimed {
305        job: u64,
306        run_id: u64,
307        runner: String,
308        at: u64,
309    },
310    Details {
311        details: JobDetails,
312    },
313    Elsewhere {
314        details: JobDetails,
315    },
316    Lost {
317        job: u64,
318        at: u64,
319    },
320    Unanswered {
321        job: u64,
322        at: u64,
323    },
324    Log {
325        job: u64,
326        lines: Vec<String>,
327        /// Where the capture resumes once these lines are stored; `None` for all but the last part of a batch.
328        position: Option<LogPosition>,
329        max_bytes: u64,
330        /// The lines' share of `LOG_BACKLOG`, given back once they are stored.
331        _backlog: OwnedSemaphorePermit,
332    },
333    Samples(Vec<SampleRecord>),
334    HostSample(api::HostSample),
335    MirrorBytes {
336        bytes: Vec<(String, u64)>,
337        until: MirrorPosition,
338        /// Whether the count committed.
339        done: oneshot::Sender<bool>,
340    },
341    CacheSizes {
342        at: u64,
343        sizes: BTreeMap<String, u64>,
344    },
345    /// Boxed: an event with a job's summary is many times the other writes' size.
346    Event(Box<Event>),
347    RecordMade(api::Record),
348    RecordSkipped {
349        job: u64,
350        why: String,
351    },
352    RecordRemoved {
353        job: u64,
354        at: u64,
355        by: api::RecordRemoval,
356    },
357    Paused {
358        repositories: Vec<String>,
359        /// Whether it committed.
360        done: oneshot::Sender<bool>,
361    },
362    Visibility {
363        repository: String,
364        visibility: api::Visibility,
365        at: u64,
366    },
367    SizingIntent {
368        change: api::SizingChange,
369        /// Its row, once it committed.
370        done: oneshot::Sender<Option<i64>>,
371    },
372    SizingResolved {
373        id: i64,
374        resolution: Resolution,
375        /// Whether it committed.
376        done: oneshot::Sender<bool>,
377    },
378    Notify {
379        change: NotifyChange,
380        /// Whether it committed.
381        done: oneshot::Sender<bool>,
382    },
383    RunnerNamed {
384        secs: u64,
385        /// Whether it committed.
386        done: oneshot::Sender<bool>,
387    },
388    Flush(oneshot::Sender<()>),
389}
390
391/// What the notifier records.
392#[derive(Debug)]
393enum NotifyChange {
394    /// Sets a target's keys and removes others.
395    Record {
396        target: String,
397        set: Vec<(String, u64)>,
398        remove: Vec<String>,
399    },
400    /// Forgets every target but these.
401    Keep(Vec<String>),
402}
403
404/// What became of a pending auto sizing change.
405#[derive(Debug, Clone, Copy, PartialEq, Eq)]
406pub enum Resolution {
407    /// The config file has it: announced as `sizing.changed`.
408    Done,
409    /// The file was not written.
410    Failed,
411    /// Left pending, and the config no longer says whether it was written: it keeps its interval and stays listed,
412    /// with what it replaced.
413    Uncertain,
414}
415
416/// Auto sizing as the history keeps it: when each profile last changed, and changes not yet known to be written.
417#[derive(Debug, Clone, Default, PartialEq, Eq)]
418pub struct AutoSizing {
419    /// The latest pending or done change of each profile, in seconds, however old.
420    pub last: BTreeMap<String, u64>,
421    /// Changes recorded before their file was written and not resolved since, by row.
422    pub pending: Vec<(i64, api::SizingChange)>,
423    /// Each profile's `cpus` as the daemon ran them, from its `profile.changed` events, oldest first: those `[history]
424    /// events` keeps, and each profile's latest however old.
425    pub cpus: Vec<sizing::CpusSet>,
426    /// Each profile's `memory` as the daemon ran it, from the same events.
427    pub memory: Vec<sizing::MemorySet>,
428    /// Each profile's slots as the daemon ran them, from the same events: one more for each slot a lowering left a
429    /// runner in until it ends.
430    pub slots: Vec<sizing::SlotsSet>,
431}
432
433/// The job history. Cloning shares it.
434#[derive(Debug, Clone)]
435pub struct Store {
436    writes: mpsc::UnboundedSender<Write>,
437    reader: Arc<Mutex<Connection>>,
438    /// Taken with `try_lock`, so one prune runs at a time; one that panicked midway left rows the next one deletes.
439    pruner: Arc<Mutex<Connection>>,
440    /// Set by `close`: a running prune stops after its batch, and none starts.
441    closed: Arc<AtomicBool>,
442    /// Host samples sent and not yet stored, oldest first; reads include them. The writer removes them once stored.
443    // Locked with `sync::lock`: every critical section is a push, a copy or a drain.
444    held: Arc<Mutex<Vec<api::HostSample>>>,
445    /// The last host sample sent, for admission to read without a query.
446    latest_host: Arc<Mutex<Option<api::HostSample>>>,
447    /// The last reading of the host's memory, which is not stored.
448    latest_host_memory: Arc<Mutex<Option<api::HostMemory>>>,
449    /// The jobs the last plan had waiting for the host's memory, and when (ms); insights read it.
450    host_waiting: Arc<Mutex<(BTreeSet<u64>, u64)>>,
451    /// Each running runner's last sample, for admission to read without a query.
452    latest_samples: Arc<Mutex<BTreeMap<String, SampleRecord>>>,
453    /// The runners whose container's cgroup the sampler saw go, until they are removed or it reads them again.
454    gone: Arc<Mutex<BTreeSet<String>>>,
455    events: broadcast::Sender<Arc<EventEnvelope>>,
456    backlog: Arc<Semaphore>,
457}
458
459impl Store {
460    /// Opens the database at `path`, creating it and its directory (mode 0700) if missing, with the schema when it
461    /// is new, and brings it to the schema this build reads. Refuses a database from before 0.0.19 or from a newer
462    /// runnerd. The time since the history's last sign of life is taken as time no runnerd ran: the jobs queued in
463    /// it leave it out of their waits.
464    pub fn open(path: &Path) -> Result<Store, StoreError> {
465        Store::open_at(path, crate::time::now_ms())
466    }
467
468    /// [`Store::open`] at `now` (ms).
469    fn open_at(path: &Path, now: u64) -> Result<Store, StoreError> {
470        if let Some(dir) = path.parent().filter(|dir| !dir.as_os_str().is_empty()) {
471            std::fs::DirBuilder::new()
472                .recursive(true)
473                .mode(0o700)
474                .create(dir)?;
475        }
476        // Owner-only, as SQLite then makes its `-wal` and `-shm` files: logs may say more than the owner wants shared.
477        std::fs::OpenOptions::new()
478            .create(true)
479            .append(true)
480            .mode(0o600)
481            .open(path)?;
482        let mut db = Connection::open(path)?;
483        configure(&db)?;
484        prepare(&mut db)?;
485        let downtime = Downtime::before(&mut db, now)?;
486        let reader = Connection::open_with_flags(
487            path,
488            OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX,
489        )?;
490        configure(&reader)?;
491        let pruner = Connection::open(path)?;
492        configure(&pruner)?;
493        let (events, _) = broadcast::channel(EVENT_BUFFER);
494        let (writes, receiver) = mpsc::unbounded_channel();
495        let thread_events = events.clone();
496        let held = Arc::new(Mutex::new(Vec::new()));
497        let thread_held = held.clone();
498        std::thread::Builder::new()
499            .name("runnerd-store".to_owned())
500            .spawn(move || write::run(db, downtime, receiver, &thread_events, &thread_held))?;
501        Ok(Store {
502            writes,
503            reader: Arc::new(Mutex::new(reader)),
504            pruner: Arc::new(Mutex::new(pruner)),
505            closed: Arc::new(AtomicBool::new(false)),
506            held,
507            latest_host: Arc::new(Mutex::new(None)),
508            latest_host_memory: Arc::new(Mutex::new(None)),
509            host_waiting: Arc::new(Mutex::new((BTreeSet::new(), 0))),
510            latest_samples: Arc::new(Mutex::new(BTreeMap::new())),
511            gone: Arc::new(Mutex::new(BTreeSet::new())),
512            events,
513            backlog: Arc::new(Semaphore::new(
514                usize::try_from(LOG_BACKLOG).unwrap_or(usize::MAX),
515            )),
516        })
517    }
518
519    fn send(&self, write: Write) {
520        // Fails only once the writer thread is gone, after a SQLite error it logged.
521        let _ = self.writes.send(write);
522    }
523
524    /// Records a queued job the repository's runners may run, which a listing answered at `seen_at` (ms) showed;
525    /// announces it the first time, and keeps the first listing's time. A job a runner claimed first keeps its state
526    /// and gets `seen_at` if no listing showed it before and this one answered before it started.
527    pub fn queued(&self, repository: &str, job: &QueuedJob, seen_at: u64) {
528        self.send(Write::Queued {
529            repository: repository.to_owned(),
530            job: job.clone(),
531            seen_at,
532        });
533    }
534
535    /// Records a runner container that started, or one an earlier daemon started and this one adopted.
536    pub fn runner_started(&self, runner: RunnerRecord) {
537        self.send(Write::RunnerStarted(runner));
538    }
539
540    /// Records that a runner's container and registration were removed at `at` (ms).
541    pub fn runner_removed(&self, runner: &str, at: u64) {
542        lock(&self.latest_samples).remove(runner);
543        lock(&self.gone).remove(runner);
544        self.send(Write::RunnerRemoved {
545            runner: runner.to_owned(),
546            at,
547        });
548    }
549
550    /// Records every runner the history holds open, except the `running` ones, removed at `at` (ms): what recovery
551    /// found gone, so it stops counting as holding a slot.
552    pub fn runners_gone(&self, running: Vec<String>, at: u64) {
553        lock(&self.latest_samples).retain(|runner, _| running.contains(runner));
554        lock(&self.gone).retain(|runner| running.contains(runner));
555        self.send(Write::RunnersGone { running, at });
556    }
557
558    /// Records that a runner took a job at `at` (ms).
559    pub fn claimed(&self, job: u64, run_id: u64, runner: &str, at: u64) {
560        self.send(Write::Claimed {
561            job,
562            run_id,
563            runner: runner.to_owned(),
564            at,
565        });
566    }
567
568    /// Records a running job's progress and steps as GitHub reports them; announces changed steps.
569    pub fn details(&self, details: JobDetails) {
570        self.send(Write::Details { details });
571    }
572
573    /// Closes a queued job that left the queue without a runner of this daemon: another machine's runner took it, or
574    /// none did and it has no start.
575    pub fn elsewhere(&self, details: JobDetails) {
576        self.send(Write::Elsewhere { details });
577    }
578
579    /// Closes a queued job that left the queue and that GitHub no longer answers for, at `at`: none of this daemon's
580    /// runners took it, so it reads as run elsewhere, with no conclusion.
581    pub fn unanswered(&self, job: u64, at: u64) {
582        self.send(Write::Unanswered { job, at });
583    }
584
585    /// Closes a job whose container is gone and that GitHub did not report completed in time.
586    pub fn lost(&self, job: u64, at: u64) {
587        self.send(Write::Lost { job, at });
588    }
589
590    /// Appends log lines to a job, up to `max_bytes` per job, and records where the capture continues. Waits while
591    /// `LOG_BACKLOG` bytes of lines are waiting to be stored; a larger batch goes in parts, each waiting its turn.
592    pub async fn log(&self, job: u64, lines: Vec<String>, position: LogPosition, max_bytes: u64) {
593        let size = |line: &String| line.len() as u64 + 1;
594        let mut lines = lines.into_iter().map(|mut line| {
595            // No line takes the whole backlog; the capture already cuts lines far shorter.
596            if size(&line) > LOG_BACKLOG {
597                let mut end = usize::try_from(LOG_BACKLOG / 2).unwrap_or(usize::MAX);
598                while !line.is_char_boundary(end) {
599                    end -= 1;
600                }
601                line.truncate(end);
602            }
603            line
604        });
605        let mut next = lines.next();
606        loop {
607            let (mut part, mut bytes) = (Vec::new(), 0);
608            while let Some(line) = next.take_if(|line| bytes + size(line) <= LOG_BACKLOG) {
609                bytes += size(&line);
610                part.push(line);
611                next = lines.next();
612            }
613            let last = next.is_none();
614            let share = u32::try_from(bytes.max(1)).unwrap_or(u32::MAX);
615            // The semaphore is never closed.
616            let Ok(backlog) = self.backlog.clone().acquire_many_owned(share).await else {
617                return;
618            };
619            self.send(Write::Log {
620                job,
621                lines: part,
622                position: last.then_some(position),
623                max_bytes,
624                _backlog: backlog,
625            });
626            if last {
627                return;
628            }
629        }
630    }
631
632    /// Records samples of runner containers.
633    pub fn samples(&self, samples: Vec<SampleRecord>) {
634        let mut latest = lock(&self.latest_samples);
635        for sample in &samples {
636            latest.insert(sample.runner.clone(), sample.clone());
637        }
638        drop(latest);
639        self.send(Write::Samples(samples));
640    }
641
642    /// Records a sample of the slice and sends it to every subscriber at once; it is stored with the next ones, a
643    /// minute's worth in one transaction, or at `flush`.
644    pub fn host_sample(&self, sample: api::HostSample) {
645        *lock(&self.latest_host) = Some(sample.clone());
646        lock(&self.held).push(sample.clone());
647        self.send(Write::HostSample(sample));
648    }
649
650    /// A runner's last sample, stored or not; `None` before its first sample or once it is removed.
651    pub fn latest_sample(&self, runner: &str) -> Option<SampleRecord> {
652        lock(&self.latest_samples).get(runner).cloned()
653    }
654
655    /// Notes that the cgroups of these runners' containers are gone (`metrics::Sampled::gone`), for admission to read
656    /// until they are removed. Kept in memory only: a restart finds such a container stopped.
657    pub fn containers_gone(&self, runners: Vec<String>) {
658        lock(&self.gone).extend(runners);
659    }
660
661    /// Notes that these runners' containers' cgroups were read (`metrics::Sampled::read`): one marked gone came back,
662    /// as a `docker restart` brings it, and holds memory again.
663    pub fn containers_read(&self, runners: &[String]) {
664        let mut gone = lock(&self.gone);
665        for runner in runners {
666            gone.remove(runner);
667        }
668    }
669
670    /// Whether a runner's container's cgroup is gone, though its slot may hold it a while yet.
671    pub fn container_gone(&self, runner: &str) -> bool {
672        lock(&self.gone).contains(runner)
673    }
674
675    /// Records what a runner reserves now, when measured admission changed it after the start.
676    pub fn runner_admission(&self, runner: &str, admission: api::Admission) {
677        self.send(Write::RunnerAdmission {
678            runner: runner.to_owned(),
679            admission,
680        });
681    }
682
683    /// Records that Docker saw the kernel kill a process in a runner's container for memory, which its samples may
684    /// have missed: one in the last interval before the container stopped.
685    pub fn runner_oom_killed(&self, runner: &str) {
686        self.send(Write::RunnerOomKilled(runner.to_owned()));
687    }
688
689    /// Records when a forced pause that cancelled a runner's run removes it if it still runs, in ms since the epoch,
690    /// so a restart keeps that deadline.
691    pub fn runner_cancel_by(&self, runner: &str, at: u64) {
692        self.send(Write::RunnerCancelBy {
693            runner: runner.to_owned(),
694            at,
695        });
696    }
697
698    /// Records why an idle runner was removed, as `scheduler::StopReason::as_str` names it.
699    pub fn runner_stopped_for(&self, runner: &str, reason: &'static str) {
700        self.send(Write::RunnerStoppedFor {
701            runner: runner.to_owned(),
702            reason,
703        });
704    }
705
706    /// Adds what the shared build cache served a runner and took from it, and records whether its job could write.
707    pub fn build_cache(&self, runner: &str, counts: api::BuildCacheCounts) {
708        self.send(Write::BuildCache {
709            runner: runner.to_owned(),
710            counts,
711        });
712    }
713
714    /// Keeps the latest reading of the host's memory for admission and the status; it is not stored.
715    pub fn host_memory(&self, memory: api::HostMemory) {
716        *lock(&self.latest_host_memory) = Some(memory);
717    }
718
719    /// The latest reading of the host's memory; `None` before the first.
720    pub fn latest_host_memory(&self) -> Option<api::HostMemory> {
721        lock(&self.latest_host_memory).clone()
722    }
723
724    /// Keeps which `jobs` the plan at `at` (ms) had waiting for memory the host lacks (ADR 0103), for insights to read
725    /// with the reading; it is not stored. A plan that finds none replaces the last one's.
726    pub fn host_waiting(&self, jobs: BTreeSet<u64>, at: u64) {
727        *lock(&self.host_waiting) = (jobs, at);
728    }
729
730    /// Adds `ms` to how long each of `jobs` waited because the host's memory could not take its runner, which
731    /// sizing does not read as a wait for a slot.
732    pub fn host_waited(&self, jobs: Vec<u64>, ms: u64) {
733        self.send(Write::HostWaited { jobs, ms });
734    }
735
736    /// Adds `ms` to how long each of `jobs` waited with no slot of its profile free while the budgets had room for
737    /// its runner: what auto sizing reads as a wait for a slot (ADR 0090).
738    pub fn slot_waited(&self, jobs: Vec<u64>, ms: u64) {
739        self.send(Write::SlotWaited { jobs, ms });
740    }
741
742    /// The last host sample sent, stored or not; `None` before the first.
743    pub fn latest_host_sample(&self) -> Option<api::HostSample> {
744        lock(&self.latest_host).clone()
745    }
746
747    /// Adds bytes the mirror served to runners, counted from its log up to `until`, and says whether that was
748    /// stored: if not, the caller counts the same lines again.
749    pub async fn mirror_bytes(&self, bytes: Vec<(String, u64)>, until: MirrorPosition) -> bool {
750        let (done, committed) = oneshot::channel();
751        self.send(Write::MirrorBytes { bytes, until, done });
752        committed.await.unwrap_or(false)
753    }
754
755    /// Records volume sizes in bytes, by name, measured at `at` (ms).
756    pub fn cache_sizes(&self, at: u64, sizes: BTreeMap<String, u64>) {
757        self.send(Write::CacheSizes { at, sizes });
758    }
759
760    /// Stores a durable event, or sends a live one, to every subscriber.
761    pub fn event(&self, event: Event) {
762        self.send(Write::Event(Box::new(event)));
763    }
764
765    /// Records that a failed job's runner was kept as `record`, and announces it.
766    pub fn record_made(&self, record: api::Record) {
767        self.send(Write::RecordMade(record));
768    }
769
770    /// Records why a failed job's runner was not kept.
771    pub fn record_skipped(&self, job: u64, why: String) {
772        self.send(Write::RecordSkipped { job, why });
773    }
774
775    /// Records that a job's record was removed at `at` (ms), and why, and announces it.
776    pub fn record_removed(&self, job: u64, at: u64, by: api::RecordRemoval) {
777        self.send(Write::RecordRemoved { job, at, by });
778    }
779
780    /// Records the paused repositories, all of them, in place of the ones recorded before. The write is sent at
781    /// once, in order with every other; the future says whether it committed.
782    pub fn paused(&self, repositories: Vec<String>) -> impl Future<Output = bool> + use<> {
783        let (done, committed) = oneshot::channel();
784        self.send(Write::Paused { repositories, done });
785        async move { committed.await.unwrap_or(false) }
786    }
787
788    /// Records that this machine named a runner for `secs`, the second its name ends in, before GitHub or Docker
789    /// knows the name: a restart names the next runner later even when nothing else recorded this one. Says whether
790    /// it committed.
791    pub async fn runner_named(&self, secs: u64) -> bool {
792        let (done, committed) = oneshot::channel();
793        self.send(Write::RunnerNamed { secs, done });
794        // Waited for on a blocking thread, as a read is: the runtime then knows the start waits for the disk, and a
795        // paused test clock does not run on meanwhile.
796        tokio::task::spawn_blocking(move || committed.blocking_recv().unwrap_or(false))
797            .await
798            .unwrap_or(false)
799    }
800
801    /// Records a repository's visibility as GitHub gave it at `at` (ms), in place of the one recorded before.
802    pub fn visibility(&self, repository: String, visibility: api::Visibility, at: u64) {
803        self.send(Write::Visibility {
804            repository,
805            visibility,
806            at,
807        });
808    }
809
810    /// Records a change auto sizing is about to write, as pending, before the file is, synced to disk. The future
811    /// has its row once it committed, `None` if it did not: then nothing may be written.
812    pub fn sizing_intent(
813        &self,
814        change: api::SizingChange,
815    ) -> impl Future<Output = Option<i64>> + use<> {
816        let (done, committed) = oneshot::channel();
817        self.send(Write::SizingIntent { change, done });
818        async move { committed.await.ok().flatten() }
819    }
820
821    /// Records what became of a pending change, synced to disk; a done one is announced as `sizing.changed` in the
822    /// same transaction. The future says whether it committed.
823    pub fn sizing_resolved(
824        &self,
825        id: i64,
826        resolution: Resolution,
827    ) -> impl Future<Output = bool> + use<> {
828        let (done, committed) = oneshot::channel();
829        self.send(Write::SizingResolved {
830            id,
831            resolution,
832            done,
833        });
834        async move { committed.await.unwrap_or(false) }
835    }
836
837    /// Records what the notifier delivered to a target: sets `set`'s keys and removes `remove`'s, in one
838    /// transaction. The future says whether it committed.
839    pub fn notify_record(
840        &self,
841        target: &str,
842        set: Vec<(String, u64)>,
843        remove: Vec<String>,
844    ) -> impl Future<Output = bool> + use<> {
845        let (done, committed) = oneshot::channel();
846        self.send(Write::Notify {
847            change: NotifyChange::Record {
848                target: target.to_owned(),
849                set,
850                remove,
851            },
852            done,
853        });
854        async move { committed.await.unwrap_or(false) }
855    }
856
857    /// Forgets what the notifier kept for every target but `targets`. The future says whether it committed.
858    pub fn notify_keep(&self, targets: Vec<String>) -> impl Future<Output = bool> + use<> {
859        let (done, committed) = oneshot::channel();
860        self.send(Write::Notify {
861            change: NotifyChange::Keep(targets),
862            done,
863        });
864        async move { committed.await.unwrap_or(false) }
865    }
866
867    /// Deletes what is older than `history` keeps, at `now` (ms), off the writer, in batches the writer's
868    /// transactions fit between. A prune while the last one runs does nothing. The handle finishes with it; dropping
869    /// it lets it run.
870    pub fn prune(&self, now: u64, history: &History) -> tokio::task::JoinHandle<()> {
871        let pruner = self.pruner.clone();
872        let closed = self.closed.clone();
873        let history = history.clone();
874        tokio::task::spawn_blocking(move || {
875            let db = match pruner.try_lock() {
876                Ok(db) => db,
877                Err(TryLockError::Poisoned(poisoned)) => poisoned.into_inner(),
878                Err(TryLockError::WouldBlock) => return,
879            };
880            let db = prune::Db {
881                connection: &db,
882                stop: &closed,
883            };
884            if !db.stop.load(Ordering::Relaxed) {
885                prune::run(&db, now, &history);
886            }
887        })
888    }
889
890    /// Stops a running prune after its batch, and any later one before it starts, so a shutdown does not wait for
891    /// one. What a stopped prune left, the next daemon's deletes. Writes go on as before.
892    pub fn close(&self) {
893        self.closed.store(true, Ordering::Relaxed);
894    }
895
896    /// Waits until every write sent before is done.
897    pub async fn flush(&self) {
898        let (done, wait) = oneshot::channel();
899        self.send(Write::Flush(done));
900        let _ = wait.await;
901    }
902
903    /// Every event from now on, durable or live. Subscribe before reading what was stored, then skip what the read
904    /// returned, and nothing is missed or sent twice.
905    pub fn subscribe(&self) -> broadcast::Receiver<Arc<EventEnvelope>> {
906        self.events.subscribe()
907    }
908
909    async fn read<T: Send + 'static>(
910        &self,
911        query: impl FnOnce(&Connection) -> rusqlite::Result<T> + Send + 'static,
912    ) -> Result<T, StoreError> {
913        let reader = self.reader.clone();
914        // A panic during a read leaves the connection usable: reads change nothing.
915        Ok(tokio::task::spawn_blocking(move || query(&lock(&reader))).await??)
916    }
917
918    /// One page of jobs, newest first.
919    pub async fn jobs(&self, query: api::JobQuery) -> Result<api::JobPage, StoreError> {
920        self.read(move |db| read::jobs(db, &query)).await
921    }
922
923    /// A job with its steps.
924    pub async fn job(&self, id: u64) -> Result<Option<api::Job>, StoreError> {
925        self.read(move |db| read::job(db, id)).await
926    }
927
928    /// A job's samples, oldest first; `None` if there is no such job.
929    pub async fn job_samples(
930        &self,
931        id: u64,
932        interval: Duration,
933    ) -> Result<Option<api::SampleSeries>, StoreError> {
934        self.read(move |db| read::job_samples(db, id, interval))
935            .await
936    }
937
938    /// Up to `limit` log lines of a job from line `from`, with their numbers.
939    pub async fn log_lines(
940        &self,
941        job: u64,
942        from: u64,
943        limit: u64,
944    ) -> Result<Vec<(u64, String)>, StoreError> {
945        self.read(move |db| read::log_lines(db, job, from, limit))
946            .await
947    }
948
949    /// Up to `limit` durable events after `after`, oldest first.
950    pub async fn events_after(
951        &self,
952        after: u64,
953        limit: u64,
954    ) -> Result<Vec<EventEnvelope>, StoreError> {
955        self.read(move |db| read::events_after(db, after, limit))
956            .await
957    }
958
959    /// The changes auto sizing wrote since `since` (ms), oldest first.
960    pub async fn sizing_changes(&self, since: u64) -> Result<Vec<api::SizingChange>, StoreError> {
961        self.read(move |db| read::sizing_changes(db, since)).await
962    }
963
964    /// When each profile last changed, and the changes still pending.
965    pub async fn auto_sizing(&self) -> Result<AutoSizing, StoreError> {
966        self.read(read::auto_sizing).await
967    }
968
969    /// The profiles and repositories as the stored events last announced them, so a restarted daemon announces
970    /// only what changed since, including what its config no longer has. Blocks on the database: for startup.
971    pub fn announced(&self) -> Result<Announced, StoreError> {
972        Ok(read::announced(&lock(&self.reader))?)
973    }
974
975    /// The repositories recorded paused, for the engine to start with. Blocks on the database: for startup.
976    pub fn paused_repositories(&self) -> Result<Vec<String>, StoreError> {
977        Ok(read::paused(&lock(&self.reader))?)
978    }
979
980    /// The slice's samples since `since` (seconds), oldest first, the ones not yet stored included.
981    pub async fn host_samples(&self, since: u64) -> Result<Vec<api::HostSample>, StoreError> {
982        // Copied before the read: the writer removes a sample only once it is stored, so each one is in the copy,
983        // the read, or both.
984        let held = lock(&self.held).clone();
985        let mut samples = self.read(move |db| read::host_samples(db, since)).await?;
986        let stored = samples.last().map_or(0, |sample| sample.at);
987        samples.extend(
988            held.into_iter()
989                .filter(|sample| sample.at >= since && sample.at > stored),
990        );
991        Ok(samples)
992    }
993
994    /// The jobs runnerd's runners started and finished from `since` until `now` (seconds).
995    pub async fn activity(&self, since: u64, now: u64) -> Result<api::Activity, StoreError> {
996        self.read(move |db| read::activity(db, since, now)).await
997    }
998
999    /// Finished jobs per repository and bucket.
1000    pub async fn stats(&self, query: api::StatsQuery, now: u64) -> Result<api::Stats, StoreError> {
1001        self.read(move |db| read::stats(db, &query, now)).await
1002    }
1003
1004    /// What sizing reads: the finished jobs queued at or after `since` (ms) and the runners since.
1005    pub async fn sizing(&self, since: u64) -> Result<sizing::History, StoreError> {
1006        self.read(move |db| read::sizing(db, since)).await
1007    }
1008
1009    /// What pricing at GitHub's rates reads: the completed and lost jobs dated at or after `since` (ms), each
1010    /// repository's visibility and since when the history holds every final job.
1011    pub async fn pricing(&self, since: u64) -> Result<api::PricingHistory, StoreError> {
1012        self.read(move |db| read::pricing(db, since)).await
1013    }
1014
1015    /// The jobs completed or lost that were queued at or after `since` (ms), as insights read them, from when the
1016    /// history holds every job, and the queued jobs the last plan had waiting for the host's memory.
1017    pub async fn insights(&self, since: u64) -> Result<crate::insights::History, StoreError> {
1018        let memory = self.latest_host_memory();
1019        let (waiting, seen) = lock(&self.host_waiting).clone();
1020        self.read(move |db| {
1021            let mut history = read::insights(db, since)?;
1022            if let Some(memory) = memory.filter(|_| !waiting.is_empty()) {
1023                let jobs = read::held_jobs(db, &waiting)?;
1024                history.host =
1025                    (!jobs.is_empty()).then_some(crate::insights::HostHold { memory, seen, jobs });
1026            }
1027            Ok(history)
1028        })
1029        .await
1030    }
1031
1032    /// The newest durable event's ID; 0 before the first.
1033    pub async fn last_event(&self) -> Result<u64, StoreError> {
1034        self.read(read::last_event).await
1035    }
1036
1037    /// What the notifier kept for each target, by target, then key.
1038    pub async fn notify_state(
1039        &self,
1040    ) -> Result<BTreeMap<String, BTreeMap<String, u64>>, StoreError> {
1041        self.read(read::notify_state).await
1042    }
1043
1044    /// The jobs queued at or after `since` (ms), whatever became of them, as warm runners learn from them.
1045    pub async fn warm(&self, since: u64) -> Result<Vec<warm::Seen>, StoreError> {
1046        self.read(move |db| read::warm(db, since)).await
1047    }
1048
1049    /// The last measurement of every volume, with its time (seconds), by name.
1050    pub async fn cache_sizes_now(&self) -> Result<Measured, StoreError> {
1051        self.read(read::latest_cache_sizes).await
1052    }
1053
1054    /// What the build cache served and took from the runners started since `since` (ms), summed.
1055    pub async fn build_cache_since(&self, since: u64) -> Result<api::BuildCacheCounts, StoreError> {
1056        self.read(move |db| read::build_cache_since(db, since))
1057            .await
1058    }
1059
1060    /// Every volume measurement since `since` (seconds).
1061    pub async fn cache_history(&self, since: u64) -> Result<Vec<api::CacheSize>, StoreError> {
1062        self.read(move |db| read::cache_history(db, since)).await
1063    }
1064
1065    /// The jobs not yet closed, for the engine after a restart.
1066    pub async fn open_jobs(&self) -> Result<Vec<OpenJob>, StoreError> {
1067        self.read(read::open_jobs).await
1068    }
1069
1070    /// The newest named job each slot ran, for the engine after a restart.
1071    pub async fn slot_jobs(&self) -> Result<Vec<SlotJob>, StoreError> {
1072        self.read(read::slot_jobs).await
1073    }
1074
1075    /// The latest second a runner was named for (`runner_named`), or any runner name starting with `prefix` carries,
1076    /// of a runner or of a job's runner: the engine names runners later after a restart, so no name it used before
1077    /// is used twice.
1078    pub async fn last_runner_second(&self, prefix: String) -> Result<Option<u64>, StoreError> {
1079        self.read(move |db| read::last_runner_second(db, &prefix))
1080            .await
1081    }
1082
1083    /// The job a runner took, if the history knows it.
1084    pub async fn runner_job(&self, runner: &str) -> Result<Option<u64>, StoreError> {
1085        let runner = runner.to_owned();
1086        self.read(move |db| read::runner_job(db, &runner)).await
1087    }
1088
1089    /// How a runner was admitted, with what it reserves since, if the history knows it.
1090    pub async fn runner_admitted(
1091        &self,
1092        runner: &str,
1093    ) -> Result<Option<api::Admission>, StoreError> {
1094        let runner = runner.to_owned();
1095        self.read(move |db| read::runner_admission(db, &runner))
1096            .await
1097    }
1098
1099    /// When a forced pause that cancelled a runner's run removes it if it still runs, in ms since the epoch, if one
1100    /// did.
1101    pub async fn runner_cancel_deadline(&self, runner: &str) -> Result<Option<u64>, StoreError> {
1102        let runner = runner.to_owned();
1103        self.read(move |db| read::runner_cancel_deadline(db, &runner))
1104            .await
1105    }
1106
1107    /// The memory cap a runner started with, as its record kept it, for when Docker cannot tell.
1108    pub async fn runner_memory_cap(&self, runner: &str) -> Result<Option<u64>, StoreError> {
1109        let runner = runner.to_owned();
1110        self.read(move |db| read::runner_memory_cap(db, &runner))
1111            .await
1112    }
1113
1114    /// Where a job's log capture continues.
1115    pub async fn log_position(&self, job: u64) -> Result<LogPosition, StoreError> {
1116        self.read(move |db| read::log_position(db, job)).await
1117    }
1118
1119    /// How far the mirror's log was counted.
1120    pub async fn mirror_log_until(&self) -> Result<MirrorPosition, StoreError> {
1121        self.read(read::mirror_log_until).await
1122    }
1123}
1124
1125/// What sizing reads from the database at `path`, opened read-only and left as it is, as `runnerd doctor` reads it
1126/// while no daemon runs; `None` when there is no database yet. In a directory it may write, SQLite keeps the
1127/// `-wal` and `-shm` files a read of a WAL database needs next to it; in one it may not, it reads the file as it is,
1128/// or refuses when the `-wal` holds changes the file does not have yet.
1129pub fn sizing_at(path: &Path, since: u64) -> Result<Option<sizing::History>, StoreError> {
1130    read_only(path, |db| sizing_of(db, since))
1131}
1132
1133/// How the jobs of the history at `path` that started from `since` (ms) ran, read as [`sizing_at`] reads, for a replay
1134/// that runs them so (`admission::simulate`): their samples, when the daemon could name each, what the host's programs
1135/// held. `overhead` is what the slice holds beside its runners, in bytes; `None` measures it from the history's busy
1136/// moments, as `read::trace` says. `named_by` is the longest a known job waits to be read: the poll interval.
1137pub fn trace_at(
1138    path: &Path,
1139    since: u64,
1140    named_by: u64,
1141    overhead: Option<u64>,
1142) -> Result<Option<Trace>, StoreError> {
1143    read_only(path, |db| {
1144        if version(db)? == 0 {
1145            return Ok(Trace::default());
1146        }
1147        Ok(read::trace(db, since, named_by, overhead)?)
1148    })
1149}
1150
1151/// When the last job of the history at `path` was queued, in ms, read as [`sizing_at`] reads; `None` with no job yet.
1152/// A replay's window counts back from it, so a backup replays its own last hours.
1153pub fn last_queued_at(path: &Path) -> Result<Option<u64>, StoreError> {
1154    let last = read_only(path, |db| {
1155        if version(db)? == 0 {
1156            return Ok(None);
1157        }
1158        Ok(db.query_row("SELECT MAX(queued_at) FROM jobs", [], |row| row.get(0))?)
1159    })?;
1160    Ok(last.flatten())
1161}
1162
1163/// What `read` finds in the database at `path`, opened read-only and left as it is; `None` when there is none.
1164fn read_only<T>(
1165    path: &Path,
1166    read: impl Fn(&Connection) -> Result<T, StoreError>,
1167) -> Result<Option<T>, StoreError> {
1168    if !path.exists() {
1169        return Ok(None);
1170    }
1171    let read = |path: &str, flags| -> Result<T, StoreError> {
1172        let db = Connection::open_with_flags(path, flags)?;
1173        db.busy_timeout(Duration::from_secs(5))?;
1174        read(&db)
1175    };
1176    let name = path.to_string_lossy();
1177    match read(&name, OpenFlags::SQLITE_OPEN_READ_ONLY) {
1178        Err(StoreError::Sqlite(error))
1179            if matches!(
1180                error.sqlite_error_code(),
1181                Some(rusqlite::ErrorCode::ReadOnly | rusqlite::ErrorCode::CannotOpen)
1182            ) =>
1183        {
1184            // No `-shm` can be made: read the file alone, unless the `-wal` holds what it lacks.
1185            let mut wal = path.as_os_str().to_owned();
1186            wal.push("-wal");
1187            let wal = PathBuf::from(wal);
1188            if std::fs::metadata(&wal).is_ok_and(|wal| wal.len() > 0) {
1189                return Err(StoreError::Unread(wal));
1190            }
1191            let uri = format!(
1192                "file:{}?mode=ro&immutable=1",
1193                name.replace('%', "%25")
1194                    .replace('?', "%3f")
1195                    .replace('#', "%23")
1196            );
1197            read(
1198                &uri,
1199                OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_URI,
1200            )
1201        }
1202        history => history,
1203    }
1204    .map(Some)
1205}
1206
1207/// What sizing reads from the database, without creating its schema; an older one reads as [`VERSION`].
1208fn sizing_of(db: &Connection, since: u64) -> Result<sizing::History, StoreError> {
1209    match version(db)? {
1210        0 => return Ok(sizing::History::default()),
1211        VERSION => {}
1212        _ => read_as_current(db)?,
1213    }
1214    let mut history = read::sizing(db, since)?;
1215    let auto = read::auto_sizing(db)?;
1216    history.memory_set = auto.memory;
1217    history.slots_set = auto.slots;
1218    history.changes = read::sizing_changes(db, since.saturating_sub(sizing::memory::HISTORY))?;
1219    Ok(history)
1220}
1221
1222/// The database's schema version: 0 for a new one, else from [`OLDEST`] to [`VERSION`]; any other is refused.
1223fn version(db: &Connection) -> Result<u32, StoreError> {
1224    let found: u32 = db.pragma_query_value(None, "user_version", |row| row.get(0))?;
1225    match found {
1226        0 | OLDEST..=VERSION => Ok(found),
1227        found if found > VERSION => Err(StoreError::Newer {
1228            found,
1229            supported: VERSION,
1230        }),
1231        found => Err(StoreError::Older {
1232            found,
1233            oldest: OLDEST,
1234        }),
1235    }
1236}
1237
1238/// Lets a read-only look at a database from before [`VERSION`], which it cannot migrate, read it as this build
1239/// does: a table it lacks becomes an empty temporary one, and a table that lacks columns a temporary view of it
1240/// that adds them, each with its default. Temporary objects come first in name lookups.
1241fn read_as_current(db: &Connection) -> Result<(), StoreError> {
1242    let current = Connection::open_in_memory()?;
1243    current.execute_batch(SCHEMA)?;
1244    let tables: Vec<(String, String)> = current
1245        .prepare(
1246            "SELECT name, sql FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'",
1247        )?
1248        .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?
1249        .collect::<rusqlite::Result<_>>()?;
1250    let columns =
1251        |db: &Connection, table: &str| -> rusqlite::Result<Vec<(String, Option<String>)>> {
1252            db.prepare("SELECT name, dflt_value FROM pragma_table_info(?1, 'main')")?
1253                .query_map([table], |row| Ok((row.get(0)?, row.get(1)?)))?
1254                .collect()
1255        };
1256    for (table, sql) in tables {
1257        let found: Vec<String> = columns(db, &table)?
1258            .into_iter()
1259            .map(|(name, _)| name)
1260            .collect();
1261        if found.is_empty() {
1262            db.execute_batch(&sql.replacen("CREATE TABLE", "CREATE TEMP TABLE", 1))?;
1263            continue;
1264        }
1265        let added: Vec<String> = columns(&current, &table)?
1266            .into_iter()
1267            .filter(|(name, _)| !found.contains(name))
1268            .map(|(name, default)| format!("{} AS {name}", default.as_deref().unwrap_or("NULL")))
1269            .collect();
1270        if !added.is_empty() {
1271            db.execute_batch(&format!(
1272                "CREATE TEMP VIEW {table} AS SELECT *, {} FROM main.{table}",
1273                added.join(", ")
1274            ))?;
1275        }
1276    }
1277    Ok(())
1278}
1279
1280/// WAL, so the API reads while the writer writes; a busy timeout instead of an immediate error.
1281fn configure(db: &Connection) -> rusqlite::Result<()> {
1282    db.pragma_update(None, "journal_mode", "wal")?;
1283    db.pragma_update(None, "synchronous", "normal")?;
1284    db.pragma_update(None, "foreign_keys", "on")?;
1285    db.busy_timeout(Duration::from_secs(5))
1286}
1287
1288/// Creates the schema in a new database, in one transaction with its version, or runs the migrations an older one
1289/// lacks, each in its own transaction with the version it brings it to.
1290fn prepare(db: &mut Connection) -> Result<(), StoreError> {
1291    let found = version(db)?;
1292    if found == 0 {
1293        let transaction = db.transaction()?;
1294        transaction.execute_batch(SCHEMA)?;
1295        transaction.pragma_update(None, "user_version", VERSION)?;
1296        transaction.commit()?;
1297        return Ok(());
1298    }
1299    for (version, migration) in MIGRATIONS.iter().filter(|(version, _)| *version > found) {
1300        let transaction = db.transaction()?;
1301        transaction.execute_batch(migration)?;
1302        transaction.pragma_update(None, "user_version", version)?;
1303        transaction.commit()?;
1304    }
1305    Ok(())
1306}
1307
1308/// The time before this daemon when no runnerd ran, in ms: from the history's last sign of life, its newest host
1309/// sample, event or open, to this daemon's start. Host samples are stored a minute's worth at a time, so a crash
1310/// may count up to a minute more. Without a sign of life, as in a new history, there is none: the history's start
1311/// clips the waits of jobs queued before it (`history_kept`).
1312#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1313struct Downtime {
1314    from: u64,
1315    until: u64,
1316}
1317
1318impl Downtime {
1319    /// The downtime before a daemon that starts at `now`; the jobs still queued in the history leave it out of
1320    /// their waits, as the ones first seen after do when they are recorded. The open is a sign of life itself, kept
1321    /// with them: a start that fails after it, and is retried, does not hand out the same downtime again.
1322    fn before(db: &mut Connection, now: u64) -> Result<Option<Downtime>, StoreError> {
1323        // Immediate: it reads, then writes. A deferred transaction that has read cannot wait for the write lock, and
1324        // fails at once while another connection, such as a dropped store's writer finishing, holds it.
1325        let transaction = db.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?;
1326        let last: Option<u64> = transaction.query_row(
1327            "SELECT max(at) FROM (SELECT max(at) AS at FROM host_samples UNION ALL SELECT max(at) FROM events \
1328             UNION ALL SELECT value FROM meta WHERE key = 'opened_at')",
1329            [],
1330            |row| row.get(0),
1331        )?;
1332        let downtime = last
1333            .filter(|last| *last < now)
1334            .map(|from| Downtime { from, until: now });
1335        if let Some(downtime) = downtime {
1336            let queued: Vec<(u64, u64)> = transaction
1337                .prepare("SELECT id, queued_at FROM jobs WHERE state = 'queued'")?
1338                .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?
1339                .collect::<rusqlite::Result<_>>()?;
1340            for (job, queued_at) in queued {
1341                transaction.execute(
1342                    "UPDATE jobs SET unseen = unseen + ?2 WHERE id = ?1",
1343                    rusqlite::params![job, downtime.of(queued_at)],
1344                )?;
1345            }
1346        }
1347        transaction.execute(
1348            "INSERT OR REPLACE INTO meta (key, value) VALUES ('opened_at', ?1)",
1349            [now],
1350        )?;
1351        transaction.commit()?;
1352        Ok(downtime)
1353    }
1354
1355    /// How much of it a job queued at `queued_at` (ms) waited through.
1356    fn of(self, queued_at: u64) -> u64 {
1357        self.until.saturating_sub(queued_at.max(self.from))
1358    }
1359
1360    /// How much of what [`Downtime::of`] gave a job queued at `queued_at` came after it started at `started_at`
1361    /// (ms), as for a job a runner that outlived the last daemon took while none ran.
1362    fn after(self, queued_at: u64, started_at: u64) -> u64 {
1363        self.of(queued_at)
1364            .saturating_sub(started_at.saturating_sub(queued_at.max(self.from)))
1365    }
1366}
1367
1368/// Logs a failed write; the writer goes on with the next one.
1369fn logged<T>(what: &str, result: rusqlite::Result<T>) -> Option<T> {
1370    result
1371        .map_err(|error| warn!("job history: cannot {what}: {error}"))
1372        .ok()
1373}
1374
1375#[cfg(test)]
1376mod tests;