Skip to main content

runnerd_core/
store.rs

1//! The job history: a SQLite database of jobs, their runners, steps, samples and logs, the events clients resume
2//! from, cache sizes, the paused repositories and where the notifier got to. Owns the schema, the one writer and
3//! the event broadcast. Writes are messages to a writer thread, so the engine never waits on the disk; reads and
4//! pruning run on Tokio's blocking pool, pruning on its own connection, a batch at a time. It decides nothing about
5//! scheduling and stores no secret: no token, no JIT config, no runner `_diag` log.
6
7use std::collections::BTreeMap;
8use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt};
9use std::path::{Path, PathBuf};
10use std::sync::atomic::{AtomicBool, Ordering};
11use std::sync::{Arc, Mutex, TryLockError};
12use std::time::Duration;
13
14use rusqlite::{Connection, OpenFlags};
15use tokio::sync::{OwnedSemaphorePermit, Semaphore, broadcast, mpsc, oneshot};
16use tracing::warn;
17
18use crate::api::{self, Event, EventEnvelope};
19use crate::config::History;
20use crate::github::{JobDetails, QueuedJob};
21use crate::observe::metrics::{CpuHeld, Events};
22use crate::scheduler::warm;
23use crate::sizing;
24use crate::sync::lock;
25
26mod prune;
27mod read;
28mod write;
29
30/// The schema a new database is created with, at [`VERSION`].
31const SCHEMA: &str = include_str!("store/schema.sql");
32
33/// The oldest schema version a database is brought up from: 0.0.19's (ADR 0073).
34const OLDEST: u32 = 15;
35
36/// The steps from [`OLDEST`] on, in order, each with the version it brings a database to. A change to the schema
37/// adds one here and makes the same change to `schema.sql`, which a new database gets instead.
38const MIGRATIONS: &[(u32, &str)] = &[
39    (
40        16,
41        "ALTER TABLE jobs ADD COLUMN unseen INTEGER NOT NULL DEFAULT 0;",
42    ),
43    (
44        17,
45        "ALTER TABLE jobs ADD COLUMN cpu_periods INTEGER;\n\
46         ALTER TABLE jobs ADD COLUMN cpu_throttled INTEGER;\n\
47         ALTER TABLE jobs ADD COLUMN cpu_stalled INTEGER;\n\
48         ALTER TABLE samples ADD COLUMN cpu_periods INTEGER;\n\
49         ALTER TABLE samples ADD COLUMN cpu_throttled INTEGER;\n\
50         ALTER TABLE samples ADD COLUMN cpu_stalled INTEGER;",
51    ),
52    (
53        18,
54        "ALTER TABLE jobs ADD COLUMN slot_waited INTEGER NOT NULL DEFAULT 0;",
55    ),
56    (
57        19,
58        "ALTER TABLE jobs ADD COLUMN seen_at INTEGER;\n\
59         ALTER TABLE runners ADD COLUMN purpose TEXT;\n\
60         ALTER TABLE runners ADD COLUMN stopped_for TEXT;",
61    ),
62];
63
64/// The schema version this build reads and writes, the database's `user_version`: its last migration's.
65const VERSION: u32 = match MIGRATIONS.last() {
66    Some((version, _)) => *version,
67    None => OLDEST,
68};
69
70/// Bytes of log lines sent and not yet stored, at most: a job's capture waits beyond it, so a noisy job or a slow
71/// disk holds back that job's log, not the daemon's memory.
72pub(crate) const LOG_BACKLOG: u64 = 8 << 20;
73
74/// Host samples the writer holds before it stores them in one transaction: a minute's worth. One transaction per
75/// sample wrote about 300 MB a day to an idle daemon's WAL. A crash loses the samples held.
76const HOST_SAMPLE_BATCH: usize = 30;
77
78/// How many events a slow client may fall behind before its stream ends; it reconnects with `Last-Event-ID`.
79const EVENT_BUFFER: usize = 4096;
80
81/// Why the database cannot be used.
82#[derive(Debug, thiserror::Error)]
83pub enum StoreError {
84    /// SQLite failed.
85    #[error("{0}")]
86    Sqlite(#[from] rusqlite::Error),
87    /// A newer runnerd wrote the database.
88    #[error(
89        "the job history database is version {found}, newer than the {supported} this runnerd reads; \
90         run a newer runnerd or move the database aside"
91    )]
92    Newer {
93        /// The database's version.
94        found: u32,
95        /// The newest version this build knows.
96        supported: u32,
97    },
98    /// A runnerd before 0.0.19 wrote the database.
99    #[error(
100        "the job history database is version {found}, from a runnerd before 0.0.19; this runnerd reads versions \
101         from {oldest} on: move the database aside"
102    )]
103    Older {
104        /// The database's version.
105        found: u32,
106        /// The oldest version this build reads.
107        oldest: u32,
108    },
109    /// The directory or file cannot be created.
110    #[error("{0}")]
111    Io(#[from] std::io::Error),
112    /// A blocking read did not finish.
113    #[error("a database read stopped: {0}")]
114    Join(#[from] tokio::task::JoinError),
115    /// The database's `-wal` file holds changes not yet in it, and this user cannot open the database the way that
116    /// reads them.
117    #[error(
118        "{} holds changes not yet in the job history, which this user cannot read; run runnerd doctor as the user \
119         the daemon runs as",
120        .0.display()
121    )]
122    Unread(PathBuf),
123}
124
125/// A runner container as the history records it.
126#[derive(Debug, Clone, PartialEq, Eq)]
127pub struct RunnerRecord {
128    /// The runner's name, also its container's.
129    pub name: String,
130    /// `owner/name`.
131    pub repository: String,
132    /// The profile it runs in.
133    pub profile: String,
134    /// 1-based.
135    pub slot: usize,
136    /// The Docker container ID.
137    pub container: String,
138    /// When the container started, in milliseconds since the epoch.
139    pub started_at: u64,
140    /// The container's memory cap in bytes.
141    pub memory_cap: Option<u64>,
142    /// The container's CPU limit in millicores.
143    pub cpu_cap: Option<u64>,
144    /// How it was admitted; `None` for one a restart adopted without a record of it.
145    pub admission: Option<api::Admission>,
146    /// Why it started: `job`, `warm` or `burst`; `None` for one a restart adopted.
147    pub purpose: Option<&'static str>,
148}
149
150/// One sample of one runner's container.
151#[derive(Debug, Clone, PartialEq, Eq)]
152pub struct SampleRecord {
153    /// The runner.
154    pub runner: String,
155    /// When, in milliseconds since the epoch.
156    pub at: u64,
157    /// Millicores over the interval before it.
158    pub cpu: u64,
159    /// CPU time so far, in microseconds.
160    pub cpu_usage: u64,
161    /// Bytes without page cache (`observe::metrics::Usage::memory`).
162    pub memory: u64,
163    /// Bytes received since the container started.
164    pub network_received: u64,
165    /// Bytes sent since the container started.
166    pub network_sent: u64,
167    /// The container's `memory.events` so far.
168    pub events: Events,
169    /// What held the container's CPU back so far.
170    pub cpu_held: CpuHeld,
171}
172
173/// The last measurement of the cache volumes.
174#[derive(Debug, Clone, Default, PartialEq, Eq)]
175pub struct Measured {
176    /// When it was taken (seconds); none before the first measurement.
177    pub at: Option<u64>,
178    /// Each volume's size in bytes, by name.
179    pub bytes: BTreeMap<String, u64>,
180}
181
182/// The profiles and repositories as the stored events last announced them: what clients resuming after those
183/// events have.
184#[derive(Debug, Clone, Default, PartialEq, Eq)]
185pub struct Announced {
186    /// The last `profile.changed` of each profile not removed since, by name.
187    pub profiles: Vec<api::ProfileStatus>,
188    /// The last `repository.changed` of each repository not removed since, by name.
189    pub repositories: Vec<api::RepositoryStatus>,
190}
191
192/// A job the history has not closed, as the engine needs it after a restart.
193#[derive(Debug, Clone, PartialEq, Eq)]
194pub struct OpenJob {
195    /// GitHub's job ID.
196    pub id: u64,
197    /// `owner/name`.
198    pub repository: String,
199    /// `Queued` or `Running`.
200    pub state: api::JobState,
201    /// Its runner, once one took it.
202    pub runner: Option<String>,
203    /// When its runner was removed, in milliseconds since the epoch.
204    pub runner_removed_at: Option<u64>,
205    /// The workflow run it belongs to.
206    pub run_id: u64,
207    /// 1 for the run's first attempt.
208    pub run_attempt: u32,
209    /// The workflow's name, when known.
210    pub workflow: Option<String>,
211    /// When GitHub queued it, in milliseconds since the epoch.
212    pub queued_at: u64,
213    /// The labels it asked for, which tell its repository's pool.
214    pub labels: Vec<String>,
215}
216
217/// The job a slot ran last, as the history has it.
218#[derive(Debug, Clone, PartialEq, Eq)]
219pub struct SlotJob {
220    /// The profile the slot is in.
221    pub profile: String,
222    /// 1-based.
223    pub slot: usize,
224    /// The job, as admission tells jobs apart.
225    pub key: crate::admission::Key,
226}
227
228/// Where a job's log capture continues: the runner's page file and the byte offset in it.
229#[derive(Debug, Clone, Copy, PartialEq, Eq)]
230pub struct LogPosition {
231    /// The page number, from 1.
232    pub page: u64,
233    /// Bytes of that page already stored as lines.
234    pub offset: u64,
235}
236
237/// How far the mirror's log was counted: the time of the last response counted, in nanoseconds since the epoch as
238/// Docker logs it, and how many responses logged at exactly that time were counted, so the next read counts the
239/// ones it was too early to see.
240#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
241pub struct MirrorPosition {
242    /// Nanoseconds since the epoch.
243    pub at: u64,
244    /// Responses logged at `at` already counted.
245    pub seen: u64,
246}
247
248/// What the writer thread is asked to do, in order.
249#[derive(Debug)]
250enum Write {
251    Queued {
252        repository: String,
253        job: QueuedJob,
254        seen_at: u64,
255    },
256    RunnerStarted(RunnerRecord),
257    RunnerAdmission {
258        runner: String,
259        admission: api::Admission,
260    },
261    RunnerOomKilled(String),
262    RunnerStoppedFor {
263        runner: String,
264        reason: &'static str,
265    },
266    HostWaited {
267        jobs: Vec<u64>,
268        ms: u64,
269    },
270    SlotWaited {
271        jobs: Vec<u64>,
272        ms: u64,
273    },
274    BuildCache {
275        runner: String,
276        counts: api::BuildCacheCounts,
277    },
278    RunnerRemoved {
279        runner: String,
280        at: u64,
281    },
282    RunnersGone {
283        running: Vec<String>,
284        at: u64,
285    },
286    Claimed {
287        job: u64,
288        run_id: u64,
289        runner: String,
290        at: u64,
291    },
292    Details {
293        details: JobDetails,
294    },
295    Elsewhere {
296        details: JobDetails,
297    },
298    Lost {
299        job: u64,
300        at: u64,
301    },
302    Log {
303        job: u64,
304        lines: Vec<String>,
305        /// Where the capture resumes once these lines are stored; `None` for all but the last part of a batch.
306        position: Option<LogPosition>,
307        max_bytes: u64,
308        /// The lines' share of `LOG_BACKLOG`, given back once they are stored.
309        _backlog: OwnedSemaphorePermit,
310    },
311    Samples(Vec<SampleRecord>),
312    HostSample(api::HostSample),
313    MirrorBytes {
314        bytes: Vec<(String, u64)>,
315        until: MirrorPosition,
316        /// Whether the count committed.
317        done: oneshot::Sender<bool>,
318    },
319    CacheSizes {
320        at: u64,
321        sizes: BTreeMap<String, u64>,
322    },
323    /// Boxed: an event with a job's summary is many times the other writes' size.
324    Event(Box<Event>),
325    RecordMade(api::Record),
326    RecordSkipped {
327        job: u64,
328        why: String,
329    },
330    RecordRemoved {
331        job: u64,
332        at: u64,
333        by: api::RecordRemoval,
334    },
335    Paused {
336        repositories: Vec<String>,
337        /// Whether it committed.
338        done: oneshot::Sender<bool>,
339    },
340    Visibility {
341        repository: String,
342        visibility: api::Visibility,
343        at: u64,
344    },
345    SizingIntent {
346        change: api::SizingChange,
347        /// Its row, once it committed.
348        done: oneshot::Sender<Option<i64>>,
349    },
350    SizingResolved {
351        id: i64,
352        resolution: Resolution,
353        /// Whether it committed.
354        done: oneshot::Sender<bool>,
355    },
356    Notify {
357        change: NotifyChange,
358        /// Whether it committed.
359        done: oneshot::Sender<bool>,
360    },
361    Flush(oneshot::Sender<()>),
362}
363
364/// What the notifier records.
365#[derive(Debug)]
366enum NotifyChange {
367    /// Sets a target's keys and removes others.
368    Record {
369        target: String,
370        set: Vec<(String, u64)>,
371        remove: Vec<String>,
372    },
373    /// Forgets every target but these.
374    Keep(Vec<String>),
375}
376
377/// What became of a pending auto sizing change.
378#[derive(Debug, Clone, Copy, PartialEq, Eq)]
379pub enum Resolution {
380    /// The config file has it: announced as `sizing.changed`.
381    Done,
382    /// The file was not written.
383    Failed,
384    /// Left pending, and the config no longer says whether it was written: it keeps its interval and stays listed,
385    /// with what it replaced.
386    Uncertain,
387}
388
389/// Auto sizing as the history keeps it: when each profile last changed, and changes not yet known to be written.
390#[derive(Debug, Clone, Default, PartialEq, Eq)]
391pub struct AutoSizing {
392    /// The latest pending or done change of each profile, in seconds, however old.
393    pub last: BTreeMap<String, u64>,
394    /// Changes recorded before their file was written and not resolved since, by row.
395    pub pending: Vec<(i64, api::SizingChange)>,
396    /// Each profile's `cpus` as the daemon ran them, from its `profile.changed` events, oldest first: those `[history]
397    /// events` keeps, and each profile's latest however old.
398    pub cpus: Vec<sizing::CpusSet>,
399    /// Each profile's `memory` as the daemon ran it, from the same events.
400    pub memory: Vec<sizing::MemorySet>,
401    /// Each profile's slots as the daemon ran them, from the same events: one more for each slot a lowering left a
402    /// runner in until it ends.
403    pub slots: Vec<sizing::SlotsSet>,
404}
405
406/// The job history. Cloning shares it.
407#[derive(Debug, Clone)]
408pub struct Store {
409    writes: mpsc::UnboundedSender<Write>,
410    reader: Arc<Mutex<Connection>>,
411    /// Taken with `try_lock`, so one prune runs at a time; one that panicked midway left rows the next one deletes.
412    pruner: Arc<Mutex<Connection>>,
413    /// Set by `close`: a running prune stops after its batch, and none starts.
414    closed: Arc<AtomicBool>,
415    /// Host samples sent and not yet stored, oldest first; reads include them. The writer removes them once stored.
416    // Locked with `sync::lock`: every critical section is a push, a copy or a drain.
417    held: Arc<Mutex<Vec<api::HostSample>>>,
418    /// The last host sample sent, for admission to read without a query.
419    latest_host: Arc<Mutex<Option<api::HostSample>>>,
420    /// The last reading of the host's memory, which is not stored.
421    latest_host_memory: Arc<Mutex<Option<api::HostMemory>>>,
422    /// Each running runner's last sample, for admission to read without a query.
423    latest_samples: Arc<Mutex<BTreeMap<String, SampleRecord>>>,
424    events: broadcast::Sender<Arc<EventEnvelope>>,
425    backlog: Arc<Semaphore>,
426}
427
428impl Store {
429    /// Opens the database at `path`, creating it and its directory (mode 0700) if missing, with the schema when it
430    /// is new, and brings it to the schema this build reads. Refuses a database from before 0.0.19 or from a newer
431    /// runnerd. The time since the history's last sign of life is taken as time no runnerd ran: the jobs queued in
432    /// it leave it out of their waits.
433    pub fn open(path: &Path) -> Result<Store, StoreError> {
434        Store::open_at(path, crate::time::now_ms())
435    }
436
437    /// [`Store::open`] at `now` (ms).
438    fn open_at(path: &Path, now: u64) -> Result<Store, StoreError> {
439        if let Some(dir) = path.parent().filter(|dir| !dir.as_os_str().is_empty()) {
440            std::fs::DirBuilder::new()
441                .recursive(true)
442                .mode(0o700)
443                .create(dir)?;
444        }
445        // Owner-only, as SQLite then makes its `-wal` and `-shm` files: logs may say more than the owner wants shared.
446        std::fs::OpenOptions::new()
447            .create(true)
448            .append(true)
449            .mode(0o600)
450            .open(path)?;
451        let mut db = Connection::open(path)?;
452        configure(&db)?;
453        prepare(&mut db)?;
454        let downtime = Downtime::before(&mut db, now)?;
455        let reader = Connection::open_with_flags(
456            path,
457            OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX,
458        )?;
459        configure(&reader)?;
460        let pruner = Connection::open(path)?;
461        configure(&pruner)?;
462        let (events, _) = broadcast::channel(EVENT_BUFFER);
463        let (writes, receiver) = mpsc::unbounded_channel();
464        let thread_events = events.clone();
465        let held = Arc::new(Mutex::new(Vec::new()));
466        let thread_held = held.clone();
467        std::thread::Builder::new()
468            .name("runnerd-store".to_owned())
469            .spawn(move || write::run(db, downtime, receiver, &thread_events, &thread_held))?;
470        Ok(Store {
471            writes,
472            reader: Arc::new(Mutex::new(reader)),
473            pruner: Arc::new(Mutex::new(pruner)),
474            closed: Arc::new(AtomicBool::new(false)),
475            held,
476            latest_host: Arc::new(Mutex::new(None)),
477            latest_host_memory: Arc::new(Mutex::new(None)),
478            latest_samples: Arc::new(Mutex::new(BTreeMap::new())),
479            events,
480            backlog: Arc::new(Semaphore::new(
481                usize::try_from(LOG_BACKLOG).unwrap_or(usize::MAX),
482            )),
483        })
484    }
485
486    fn send(&self, write: Write) {
487        // Fails only once the writer thread is gone, after a SQLite error it logged.
488        let _ = self.writes.send(write);
489    }
490
491    /// Records a queued job the repository's runners may run, which a listing answered at `seen_at` (ms) showed;
492    /// announces it the first time, and keeps the first listing's time. A job a runner claimed first keeps its state
493    /// and gets `seen_at` if no listing showed it before and this one answered before it started.
494    pub fn queued(&self, repository: &str, job: &QueuedJob, seen_at: u64) {
495        self.send(Write::Queued {
496            repository: repository.to_owned(),
497            job: job.clone(),
498            seen_at,
499        });
500    }
501
502    /// Records a runner container that started, or one an earlier daemon started and this one adopted.
503    pub fn runner_started(&self, runner: RunnerRecord) {
504        self.send(Write::RunnerStarted(runner));
505    }
506
507    /// Records that a runner's container and registration were removed at `at` (ms).
508    pub fn runner_removed(&self, runner: &str, at: u64) {
509        lock(&self.latest_samples).remove(runner);
510        self.send(Write::RunnerRemoved {
511            runner: runner.to_owned(),
512            at,
513        });
514    }
515
516    /// Records every runner the history holds open, except the `running` ones, removed at `at` (ms): what recovery
517    /// found gone, so it stops counting as holding a slot.
518    pub fn runners_gone(&self, running: Vec<String>, at: u64) {
519        lock(&self.latest_samples).retain(|runner, _| running.contains(runner));
520        self.send(Write::RunnersGone { running, at });
521    }
522
523    /// Records that a runner took a job at `at` (ms).
524    pub fn claimed(&self, job: u64, run_id: u64, runner: &str, at: u64) {
525        self.send(Write::Claimed {
526            job,
527            run_id,
528            runner: runner.to_owned(),
529            at,
530        });
531    }
532
533    /// Records a running job's progress and steps as GitHub reports them; announces changed steps.
534    pub fn details(&self, details: JobDetails) {
535        self.send(Write::Details { details });
536    }
537
538    /// Closes a queued job that left the queue without a runner of this daemon: another machine's runner took it, or
539    /// none did and it has no start.
540    pub fn elsewhere(&self, details: JobDetails) {
541        self.send(Write::Elsewhere { details });
542    }
543
544    /// Closes a job whose container is gone and that GitHub did not report completed in time.
545    pub fn lost(&self, job: u64, at: u64) {
546        self.send(Write::Lost { job, at });
547    }
548
549    /// Appends log lines to a job, up to `max_bytes` per job, and records where the capture continues. Waits while
550    /// `LOG_BACKLOG` bytes of lines are waiting to be stored; a larger batch goes in parts, each waiting its turn.
551    pub async fn log(&self, job: u64, lines: Vec<String>, position: LogPosition, max_bytes: u64) {
552        let size = |line: &String| line.len() as u64 + 1;
553        let mut lines = lines.into_iter().map(|mut line| {
554            // No line takes the whole backlog; the capture already cuts lines far shorter.
555            if size(&line) > LOG_BACKLOG {
556                let mut end = usize::try_from(LOG_BACKLOG / 2).unwrap_or(usize::MAX);
557                while !line.is_char_boundary(end) {
558                    end -= 1;
559                }
560                line.truncate(end);
561            }
562            line
563        });
564        let mut next = lines.next();
565        loop {
566            let (mut part, mut bytes) = (Vec::new(), 0);
567            while let Some(line) = next.take_if(|line| bytes + size(line) <= LOG_BACKLOG) {
568                bytes += size(&line);
569                part.push(line);
570                next = lines.next();
571            }
572            let last = next.is_none();
573            let share = u32::try_from(bytes.max(1)).unwrap_or(u32::MAX);
574            // The semaphore is never closed.
575            let Ok(backlog) = self.backlog.clone().acquire_many_owned(share).await else {
576                return;
577            };
578            self.send(Write::Log {
579                job,
580                lines: part,
581                position: last.then_some(position),
582                max_bytes,
583                _backlog: backlog,
584            });
585            if last {
586                return;
587            }
588        }
589    }
590
591    /// Records samples of runner containers.
592    pub fn samples(&self, samples: Vec<SampleRecord>) {
593        let mut latest = lock(&self.latest_samples);
594        for sample in &samples {
595            latest.insert(sample.runner.clone(), sample.clone());
596        }
597        drop(latest);
598        self.send(Write::Samples(samples));
599    }
600
601    /// Records a sample of the slice and sends it to every subscriber at once; it is stored with the next ones, a
602    /// minute's worth in one transaction, or at `flush`.
603    pub fn host_sample(&self, sample: api::HostSample) {
604        *lock(&self.latest_host) = Some(sample.clone());
605        lock(&self.held).push(sample.clone());
606        self.send(Write::HostSample(sample));
607    }
608
609    /// A runner's last sample, stored or not; `None` before its first sample or once it is removed.
610    pub fn latest_sample(&self, runner: &str) -> Option<SampleRecord> {
611        lock(&self.latest_samples).get(runner).cloned()
612    }
613
614    /// Records what a runner reserves now, when measured admission changed it after the start.
615    pub fn runner_admission(&self, runner: &str, admission: api::Admission) {
616        self.send(Write::RunnerAdmission {
617            runner: runner.to_owned(),
618            admission,
619        });
620    }
621
622    /// Records that Docker saw the kernel kill a process in a runner's container for memory, which its samples may
623    /// have missed: one in the last interval before the container stopped.
624    pub fn runner_oom_killed(&self, runner: &str) {
625        self.send(Write::RunnerOomKilled(runner.to_owned()));
626    }
627
628    /// Records why an idle runner was removed, as `scheduler::StopReason::as_str` names it.
629    pub fn runner_stopped_for(&self, runner: &str, reason: &'static str) {
630        self.send(Write::RunnerStoppedFor {
631            runner: runner.to_owned(),
632            reason,
633        });
634    }
635
636    /// Adds what the shared build cache served a runner and took from it, and records whether its job could write.
637    pub fn build_cache(&self, runner: &str, counts: api::BuildCacheCounts) {
638        self.send(Write::BuildCache {
639            runner: runner.to_owned(),
640            counts,
641        });
642    }
643
644    /// Keeps the latest reading of the host's memory for admission and the status; it is not stored.
645    pub fn host_memory(&self, memory: api::HostMemory) {
646        *lock(&self.latest_host_memory) = Some(memory);
647    }
648
649    /// The latest reading of the host's memory; `None` before the first.
650    pub fn latest_host_memory(&self) -> Option<api::HostMemory> {
651        lock(&self.latest_host_memory).clone()
652    }
653
654    /// Adds `ms` to how long each of `jobs` waited because the host's memory could not take its runner, which
655    /// sizing does not read as a wait for a slot.
656    pub fn host_waited(&self, jobs: Vec<u64>, ms: u64) {
657        self.send(Write::HostWaited { jobs, ms });
658    }
659
660    /// Adds `ms` to how long each of `jobs` waited with no slot of its profile free while the budgets had room for
661    /// its runner: what auto sizing reads as a wait for a slot (ADR 0090).
662    pub fn slot_waited(&self, jobs: Vec<u64>, ms: u64) {
663        self.send(Write::SlotWaited { jobs, ms });
664    }
665
666    /// The last host sample sent, stored or not; `None` before the first.
667    pub fn latest_host_sample(&self) -> Option<api::HostSample> {
668        lock(&self.latest_host).clone()
669    }
670
671    /// Adds bytes the mirror served to runners, counted from its log up to `until`, and says whether that was
672    /// stored: if not, the caller counts the same lines again.
673    pub async fn mirror_bytes(&self, bytes: Vec<(String, u64)>, until: MirrorPosition) -> bool {
674        let (done, committed) = oneshot::channel();
675        self.send(Write::MirrorBytes { bytes, until, done });
676        committed.await.unwrap_or(false)
677    }
678
679    /// Records volume sizes in bytes, by name, measured at `at` (ms).
680    pub fn cache_sizes(&self, at: u64, sizes: BTreeMap<String, u64>) {
681        self.send(Write::CacheSizes { at, sizes });
682    }
683
684    /// Stores a durable event, or sends a live one, to every subscriber.
685    pub fn event(&self, event: Event) {
686        self.send(Write::Event(Box::new(event)));
687    }
688
689    /// Records that a failed job's runner was kept as `record`, and announces it.
690    pub fn record_made(&self, record: api::Record) {
691        self.send(Write::RecordMade(record));
692    }
693
694    /// Records why a failed job's runner was not kept.
695    pub fn record_skipped(&self, job: u64, why: String) {
696        self.send(Write::RecordSkipped { job, why });
697    }
698
699    /// Records that a job's record was removed at `at` (ms), and why, and announces it.
700    pub fn record_removed(&self, job: u64, at: u64, by: api::RecordRemoval) {
701        self.send(Write::RecordRemoved { job, at, by });
702    }
703
704    /// Records the paused repositories, all of them, in place of the ones recorded before. The write is sent at
705    /// once, in order with every other; the future says whether it committed.
706    pub fn paused(&self, repositories: Vec<String>) -> impl Future<Output = bool> + use<> {
707        let (done, committed) = oneshot::channel();
708        self.send(Write::Paused { repositories, done });
709        async move { committed.await.unwrap_or(false) }
710    }
711
712    /// Records a repository's visibility as GitHub gave it at `at` (ms), in place of the one recorded before.
713    pub fn visibility(&self, repository: String, visibility: api::Visibility, at: u64) {
714        self.send(Write::Visibility {
715            repository,
716            visibility,
717            at,
718        });
719    }
720
721    /// Records a change auto sizing is about to write, as pending, before the file is, synced to disk. The future
722    /// has its row once it committed, `None` if it did not: then nothing may be written.
723    pub fn sizing_intent(
724        &self,
725        change: api::SizingChange,
726    ) -> impl Future<Output = Option<i64>> + use<> {
727        let (done, committed) = oneshot::channel();
728        self.send(Write::SizingIntent { change, done });
729        async move { committed.await.ok().flatten() }
730    }
731
732    /// Records what became of a pending change, synced to disk; a done one is announced as `sizing.changed` in the
733    /// same transaction. The future says whether it committed.
734    pub fn sizing_resolved(
735        &self,
736        id: i64,
737        resolution: Resolution,
738    ) -> impl Future<Output = bool> + use<> {
739        let (done, committed) = oneshot::channel();
740        self.send(Write::SizingResolved {
741            id,
742            resolution,
743            done,
744        });
745        async move { committed.await.unwrap_or(false) }
746    }
747
748    /// Records what the notifier delivered to a target: sets `set`'s keys and removes `remove`'s, in one
749    /// transaction. The future says whether it committed.
750    pub fn notify_record(
751        &self,
752        target: &str,
753        set: Vec<(String, u64)>,
754        remove: Vec<String>,
755    ) -> impl Future<Output = bool> + use<> {
756        let (done, committed) = oneshot::channel();
757        self.send(Write::Notify {
758            change: NotifyChange::Record {
759                target: target.to_owned(),
760                set,
761                remove,
762            },
763            done,
764        });
765        async move { committed.await.unwrap_or(false) }
766    }
767
768    /// Forgets what the notifier kept for every target but `targets`. The future says whether it committed.
769    pub fn notify_keep(&self, targets: Vec<String>) -> impl Future<Output = bool> + use<> {
770        let (done, committed) = oneshot::channel();
771        self.send(Write::Notify {
772            change: NotifyChange::Keep(targets),
773            done,
774        });
775        async move { committed.await.unwrap_or(false) }
776    }
777
778    /// Deletes what is older than `history` keeps, at `now` (ms), off the writer, in batches the writer's
779    /// transactions fit between. A prune while the last one runs does nothing. The handle finishes with it; dropping
780    /// it lets it run.
781    pub fn prune(&self, now: u64, history: &History) -> tokio::task::JoinHandle<()> {
782        let pruner = self.pruner.clone();
783        let closed = self.closed.clone();
784        let history = history.clone();
785        tokio::task::spawn_blocking(move || {
786            let db = match pruner.try_lock() {
787                Ok(db) => db,
788                Err(TryLockError::Poisoned(poisoned)) => poisoned.into_inner(),
789                Err(TryLockError::WouldBlock) => return,
790            };
791            let db = prune::Db {
792                connection: &db,
793                stop: &closed,
794            };
795            if !db.stop.load(Ordering::Relaxed) {
796                prune::run(&db, now, &history);
797            }
798        })
799    }
800
801    /// Stops a running prune after its batch, and any later one before it starts, so a shutdown does not wait for
802    /// one. What a stopped prune left, the next daemon's deletes. Writes go on as before.
803    pub fn close(&self) {
804        self.closed.store(true, Ordering::Relaxed);
805    }
806
807    /// Waits until every write sent before is done.
808    pub async fn flush(&self) {
809        let (done, wait) = oneshot::channel();
810        self.send(Write::Flush(done));
811        let _ = wait.await;
812    }
813
814    /// Every event from now on, durable or live. Subscribe before reading what was stored, then skip what the read
815    /// returned, and nothing is missed or sent twice.
816    pub fn subscribe(&self) -> broadcast::Receiver<Arc<EventEnvelope>> {
817        self.events.subscribe()
818    }
819
820    async fn read<T: Send + 'static>(
821        &self,
822        query: impl FnOnce(&Connection) -> rusqlite::Result<T> + Send + 'static,
823    ) -> Result<T, StoreError> {
824        let reader = self.reader.clone();
825        // A panic during a read leaves the connection usable: reads change nothing.
826        Ok(tokio::task::spawn_blocking(move || query(&lock(&reader))).await??)
827    }
828
829    /// One page of jobs, newest first.
830    pub async fn jobs(&self, query: api::JobQuery) -> Result<api::JobPage, StoreError> {
831        self.read(move |db| read::jobs(db, &query)).await
832    }
833
834    /// A job with its steps.
835    pub async fn job(&self, id: u64) -> Result<Option<api::Job>, StoreError> {
836        self.read(move |db| read::job(db, id)).await
837    }
838
839    /// A job's samples, oldest first; `None` if there is no such job.
840    pub async fn job_samples(
841        &self,
842        id: u64,
843        interval: Duration,
844    ) -> Result<Option<api::SampleSeries>, StoreError> {
845        self.read(move |db| read::job_samples(db, id, interval))
846            .await
847    }
848
849    /// Up to `limit` log lines of a job from line `from`, with their numbers.
850    pub async fn log_lines(
851        &self,
852        job: u64,
853        from: u64,
854        limit: u64,
855    ) -> Result<Vec<(u64, String)>, StoreError> {
856        self.read(move |db| read::log_lines(db, job, from, limit))
857            .await
858    }
859
860    /// Up to `limit` durable events after `after`, oldest first.
861    pub async fn events_after(
862        &self,
863        after: u64,
864        limit: u64,
865    ) -> Result<Vec<EventEnvelope>, StoreError> {
866        self.read(move |db| read::events_after(db, after, limit))
867            .await
868    }
869
870    /// The changes auto sizing wrote since `since` (ms), oldest first.
871    pub async fn sizing_changes(&self, since: u64) -> Result<Vec<api::SizingChange>, StoreError> {
872        self.read(move |db| read::sizing_changes(db, since)).await
873    }
874
875    /// When each profile last changed, and the changes still pending.
876    pub async fn auto_sizing(&self) -> Result<AutoSizing, StoreError> {
877        self.read(read::auto_sizing).await
878    }
879
880    /// The profiles and repositories as the stored events last announced them, so a restarted daemon announces
881    /// only what changed since, including what its config no longer has. Blocks on the database: for startup.
882    pub fn announced(&self) -> Result<Announced, StoreError> {
883        Ok(read::announced(&lock(&self.reader))?)
884    }
885
886    /// The repositories recorded paused, for the engine to start with. Blocks on the database: for startup.
887    pub fn paused_repositories(&self) -> Result<Vec<String>, StoreError> {
888        Ok(read::paused(&lock(&self.reader))?)
889    }
890
891    /// The slice's samples since `since` (seconds), oldest first, the ones not yet stored included.
892    pub async fn host_samples(&self, since: u64) -> Result<Vec<api::HostSample>, StoreError> {
893        // Copied before the read: the writer removes a sample only once it is stored, so each one is in the copy,
894        // the read, or both.
895        let held = lock(&self.held).clone();
896        let mut samples = self.read(move |db| read::host_samples(db, since)).await?;
897        let stored = samples.last().map_or(0, |sample| sample.at);
898        samples.extend(
899            held.into_iter()
900                .filter(|sample| sample.at >= since && sample.at > stored),
901        );
902        Ok(samples)
903    }
904
905    /// The jobs runnerd's runners started and finished from `since` until `now` (seconds).
906    pub async fn activity(&self, since: u64, now: u64) -> Result<api::Activity, StoreError> {
907        self.read(move |db| read::activity(db, since, now)).await
908    }
909
910    /// Finished jobs per repository and bucket.
911    pub async fn stats(&self, query: api::StatsQuery, now: u64) -> Result<api::Stats, StoreError> {
912        self.read(move |db| read::stats(db, &query, now)).await
913    }
914
915    /// What sizing reads: the finished jobs queued at or after `since` (ms) and the runners since.
916    pub async fn sizing(&self, since: u64) -> Result<sizing::History, StoreError> {
917        self.read(move |db| read::sizing(db, since)).await
918    }
919
920    /// What pricing at GitHub's rates reads: the completed and lost jobs dated at or after `since` (ms), each
921    /// repository's visibility and since when the history holds every final job.
922    pub async fn pricing(&self, since: u64) -> Result<api::PricingHistory, StoreError> {
923        self.read(move |db| read::pricing(db, since)).await
924    }
925
926    /// The jobs completed or lost that were queued at or after `since` (ms), as insights read them, and from when the
927    /// history holds every job.
928    pub async fn insights(&self, since: u64) -> Result<crate::insights::History, StoreError> {
929        self.read(move |db| read::insights(db, since)).await
930    }
931
932    /// The newest durable event's ID; 0 before the first.
933    pub async fn last_event(&self) -> Result<u64, StoreError> {
934        self.read(read::last_event).await
935    }
936
937    /// What the notifier kept for each target, by target, then key.
938    pub async fn notify_state(
939        &self,
940    ) -> Result<BTreeMap<String, BTreeMap<String, u64>>, StoreError> {
941        self.read(read::notify_state).await
942    }
943
944    /// The jobs queued at or after `since` (ms), whatever became of them, as warm runners learn from them.
945    pub async fn warm(&self, since: u64) -> Result<Vec<warm::Seen>, StoreError> {
946        self.read(move |db| read::warm(db, since)).await
947    }
948
949    /// The last measurement of every volume, with its time (seconds), by name.
950    pub async fn cache_sizes_now(&self) -> Result<Measured, StoreError> {
951        self.read(read::latest_cache_sizes).await
952    }
953
954    /// What the build cache served and took from the runners started since `since` (ms), summed.
955    pub async fn build_cache_since(&self, since: u64) -> Result<api::BuildCacheCounts, StoreError> {
956        self.read(move |db| read::build_cache_since(db, since))
957            .await
958    }
959
960    /// Every volume measurement since `since` (seconds).
961    pub async fn cache_history(&self, since: u64) -> Result<Vec<api::CacheSize>, StoreError> {
962        self.read(move |db| read::cache_history(db, since)).await
963    }
964
965    /// The jobs not yet closed, for the engine after a restart.
966    pub async fn open_jobs(&self) -> Result<Vec<OpenJob>, StoreError> {
967        self.read(read::open_jobs).await
968    }
969
970    /// The newest named job each slot ran, for the engine after a restart.
971    pub async fn slot_jobs(&self) -> Result<Vec<SlotJob>, StoreError> {
972        self.read(read::slot_jobs).await
973    }
974
975    /// The latest second any runner name starting with `prefix` carries, of a runner or of a job's runner: the
976    /// engine names runners later after a restart, so no name the history holds is used twice.
977    pub async fn last_runner_second(&self, prefix: String) -> Result<Option<u64>, StoreError> {
978        self.read(move |db| read::last_runner_second(db, &prefix))
979            .await
980    }
981
982    /// The job a runner took, if the history knows it.
983    pub async fn runner_job(&self, runner: &str) -> Result<Option<u64>, StoreError> {
984        let runner = runner.to_owned();
985        self.read(move |db| read::runner_job(db, &runner)).await
986    }
987
988    /// How a runner was admitted, with what it reserves since, if the history knows it.
989    pub async fn runner_admitted(
990        &self,
991        runner: &str,
992    ) -> Result<Option<api::Admission>, StoreError> {
993        let runner = runner.to_owned();
994        self.read(move |db| read::runner_admission(db, &runner))
995            .await
996    }
997
998    /// Where a job's log capture continues.
999    pub async fn log_position(&self, job: u64) -> Result<LogPosition, StoreError> {
1000        self.read(move |db| read::log_position(db, job)).await
1001    }
1002
1003    /// How far the mirror's log was counted.
1004    pub async fn mirror_log_until(&self) -> Result<MirrorPosition, StoreError> {
1005        self.read(read::mirror_log_until).await
1006    }
1007}
1008
1009/// What sizing reads from the database at `path`, opened read-only and left as it is, as `runnerd doctor` reads it
1010/// while no daemon runs; `None` when there is no database yet. In a directory it may write, SQLite keeps the
1011/// `-wal` and `-shm` files a read of a WAL database needs next to it; in one it may not, it reads the file as it is,
1012/// or refuses when the `-wal` holds changes the file does not have yet.
1013pub fn sizing_at(path: &Path, since: u64) -> Result<Option<sizing::History>, StoreError> {
1014    if !path.exists() {
1015        return Ok(None);
1016    }
1017    let read = |path: &str, flags| -> Result<sizing::History, StoreError> {
1018        let db = Connection::open_with_flags(path, flags)?;
1019        db.busy_timeout(Duration::from_secs(5))?;
1020        sizing_of(&db, since)
1021    };
1022    let name = path.to_string_lossy();
1023    match read(&name, OpenFlags::SQLITE_OPEN_READ_ONLY) {
1024        Err(StoreError::Sqlite(error))
1025            if matches!(
1026                error.sqlite_error_code(),
1027                Some(rusqlite::ErrorCode::ReadOnly | rusqlite::ErrorCode::CannotOpen)
1028            ) =>
1029        {
1030            // No `-shm` can be made: read the file alone, unless the `-wal` holds what it lacks.
1031            let mut wal = path.as_os_str().to_owned();
1032            wal.push("-wal");
1033            let wal = PathBuf::from(wal);
1034            if std::fs::metadata(&wal).is_ok_and(|wal| wal.len() > 0) {
1035                return Err(StoreError::Unread(wal));
1036            }
1037            let uri = format!(
1038                "file:{}?mode=ro&immutable=1",
1039                name.replace('%', "%25")
1040                    .replace('?', "%3f")
1041                    .replace('#', "%23")
1042            );
1043            read(
1044                &uri,
1045                OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_URI,
1046            )
1047        }
1048        history => history,
1049    }
1050    .map(Some)
1051}
1052
1053/// What sizing reads from the database, without creating its schema; an older one reads as [`VERSION`].
1054fn sizing_of(db: &Connection, since: u64) -> Result<sizing::History, StoreError> {
1055    match version(db)? {
1056        0 => return Ok(sizing::History::default()),
1057        VERSION => {}
1058        _ => read_as_current(db)?,
1059    }
1060    let mut history = read::sizing(db, since)?;
1061    let auto = read::auto_sizing(db)?;
1062    history.memory_set = auto.memory;
1063    history.slots_set = auto.slots;
1064    history.changes = read::sizing_changes(db, since.saturating_sub(sizing::memory::HISTORY))?;
1065    Ok(history)
1066}
1067
1068/// The database's schema version: 0 for a new one, else from [`OLDEST`] to [`VERSION`]; any other is refused.
1069fn version(db: &Connection) -> Result<u32, StoreError> {
1070    let found: u32 = db.pragma_query_value(None, "user_version", |row| row.get(0))?;
1071    match found {
1072        0 | OLDEST..=VERSION => Ok(found),
1073        found if found > VERSION => Err(StoreError::Newer {
1074            found,
1075            supported: VERSION,
1076        }),
1077        found => Err(StoreError::Older {
1078            found,
1079            oldest: OLDEST,
1080        }),
1081    }
1082}
1083
1084/// Lets a read-only look at a database from before [`VERSION`], which it cannot migrate, read it as this build
1085/// does: a table it lacks becomes an empty temporary one, and a table that lacks columns a temporary view of it
1086/// that adds them, each with its default. Temporary objects come first in name lookups.
1087fn read_as_current(db: &Connection) -> Result<(), StoreError> {
1088    let current = Connection::open_in_memory()?;
1089    current.execute_batch(SCHEMA)?;
1090    let tables: Vec<(String, String)> = current
1091        .prepare(
1092            "SELECT name, sql FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'",
1093        )?
1094        .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?
1095        .collect::<rusqlite::Result<_>>()?;
1096    let columns =
1097        |db: &Connection, table: &str| -> rusqlite::Result<Vec<(String, Option<String>)>> {
1098            db.prepare("SELECT name, dflt_value FROM pragma_table_info(?1, 'main')")?
1099                .query_map([table], |row| Ok((row.get(0)?, row.get(1)?)))?
1100                .collect()
1101        };
1102    for (table, sql) in tables {
1103        let found: Vec<String> = columns(db, &table)?
1104            .into_iter()
1105            .map(|(name, _)| name)
1106            .collect();
1107        if found.is_empty() {
1108            db.execute_batch(&sql.replacen("CREATE TABLE", "CREATE TEMP TABLE", 1))?;
1109            continue;
1110        }
1111        let added: Vec<String> = columns(&current, &table)?
1112            .into_iter()
1113            .filter(|(name, _)| !found.contains(name))
1114            .map(|(name, default)| format!("{} AS {name}", default.as_deref().unwrap_or("NULL")))
1115            .collect();
1116        if !added.is_empty() {
1117            db.execute_batch(&format!(
1118                "CREATE TEMP VIEW {table} AS SELECT *, {} FROM main.{table}",
1119                added.join(", ")
1120            ))?;
1121        }
1122    }
1123    Ok(())
1124}
1125
1126/// WAL, so the API reads while the writer writes; a busy timeout instead of an immediate error.
1127fn configure(db: &Connection) -> rusqlite::Result<()> {
1128    db.pragma_update(None, "journal_mode", "wal")?;
1129    db.pragma_update(None, "synchronous", "normal")?;
1130    db.pragma_update(None, "foreign_keys", "on")?;
1131    db.busy_timeout(Duration::from_secs(5))
1132}
1133
1134/// Creates the schema in a new database, in one transaction with its version, or runs the migrations an older one
1135/// lacks, each in its own transaction with the version it brings it to.
1136fn prepare(db: &mut Connection) -> Result<(), StoreError> {
1137    let found = version(db)?;
1138    if found == 0 {
1139        let transaction = db.transaction()?;
1140        transaction.execute_batch(SCHEMA)?;
1141        transaction.pragma_update(None, "user_version", VERSION)?;
1142        transaction.commit()?;
1143        return Ok(());
1144    }
1145    for (version, migration) in MIGRATIONS.iter().filter(|(version, _)| *version > found) {
1146        let transaction = db.transaction()?;
1147        transaction.execute_batch(migration)?;
1148        transaction.pragma_update(None, "user_version", version)?;
1149        transaction.commit()?;
1150    }
1151    Ok(())
1152}
1153
1154/// The time before this daemon when no runnerd ran, in ms: from the history's last sign of life, its newest host
1155/// sample, event or open, to this daemon's start. Host samples are stored a minute's worth at a time, so a crash
1156/// may count up to a minute more. Without a sign of life, as in a new history, there is none: the history's start
1157/// clips the waits of jobs queued before it (`history_kept`).
1158#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1159struct Downtime {
1160    from: u64,
1161    until: u64,
1162}
1163
1164impl Downtime {
1165    /// The downtime before a daemon that starts at `now`; the jobs still queued in the history leave it out of
1166    /// their waits, as the ones first seen after do when they are recorded. The open is a sign of life itself, kept
1167    /// with them: a start that fails after it, and is retried, does not hand out the same downtime again.
1168    fn before(db: &mut Connection, now: u64) -> Result<Option<Downtime>, StoreError> {
1169        // Immediate: it reads, then writes. A deferred transaction that has read cannot wait for the write lock, and
1170        // fails at once while another connection, such as a dropped store's writer finishing, holds it.
1171        let transaction = db.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?;
1172        let last: Option<u64> = transaction.query_row(
1173            "SELECT max(at) FROM (SELECT max(at) AS at FROM host_samples UNION ALL SELECT max(at) FROM events \
1174             UNION ALL SELECT value FROM meta WHERE key = 'opened_at')",
1175            [],
1176            |row| row.get(0),
1177        )?;
1178        let downtime = last
1179            .filter(|last| *last < now)
1180            .map(|from| Downtime { from, until: now });
1181        if let Some(downtime) = downtime {
1182            let queued: Vec<(u64, u64)> = transaction
1183                .prepare("SELECT id, queued_at FROM jobs WHERE state = 'queued'")?
1184                .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?
1185                .collect::<rusqlite::Result<_>>()?;
1186            for (job, queued_at) in queued {
1187                transaction.execute(
1188                    "UPDATE jobs SET unseen = unseen + ?2 WHERE id = ?1",
1189                    rusqlite::params![job, downtime.of(queued_at)],
1190                )?;
1191            }
1192        }
1193        transaction.execute(
1194            "INSERT OR REPLACE INTO meta (key, value) VALUES ('opened_at', ?1)",
1195            [now],
1196        )?;
1197        transaction.commit()?;
1198        Ok(downtime)
1199    }
1200
1201    /// How much of it a job queued at `queued_at` (ms) waited through.
1202    fn of(self, queued_at: u64) -> u64 {
1203        self.until.saturating_sub(queued_at.max(self.from))
1204    }
1205
1206    /// How much of what [`Downtime::of`] gave a job queued at `queued_at` came after it started at `started_at`
1207    /// (ms), as for a job a runner that outlived the last daemon took while none ran.
1208    fn after(self, queued_at: u64, started_at: u64) -> u64 {
1209        self.of(queued_at)
1210            .saturating_sub(started_at.saturating_sub(queued_at.max(self.from)))
1211    }
1212}
1213
1214/// Logs a failed write; the writer goes on with the next one.
1215fn logged<T>(what: &str, result: rusqlite::Result<T>) -> Option<T> {
1216    result
1217        .map_err(|error| warn!("job history: cannot {what}: {error}"))
1218        .ok()
1219}
1220
1221#[cfg(test)]
1222mod tests;