Skip to main content

runnerd_core/
store.rs

1//! The job history: a SQLite database of jobs, their runners, steps, samples and logs, the events clients resume
2//! from, cache sizes, the paused repositories and where the notifier got to. Owns the schema, the one writer and
3//! the event broadcast. Writes are messages to a writer thread, so the engine never waits on the disk; reads and
4//! pruning run on Tokio's blocking pool, pruning on its own connection, a batch at a time. It decides nothing about
5//! scheduling and stores no secret: no token, no JIT config, no runner `_diag` log.
6
7use std::collections::BTreeMap;
8use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt};
9use std::path::{Path, PathBuf};
10use std::sync::atomic::{AtomicBool, Ordering};
11use std::sync::{Arc, Mutex, TryLockError};
12use std::time::Duration;
13
14use rusqlite::{Connection, OpenFlags};
15use tokio::sync::{OwnedSemaphorePermit, Semaphore, broadcast, mpsc, oneshot};
16use tracing::warn;
17
18use crate::api::{self, Event, EventEnvelope};
19use crate::config::History;
20use crate::github::{JobDetails, QueuedJob};
21use crate::observe::metrics::{CpuHeld, Events};
22use crate::scheduler::warm;
23use crate::sizing;
24use crate::sync::lock;
25
26mod prune;
27mod read;
28mod write;
29
30/// The schema a new database is created with, at [`VERSION`].
31const SCHEMA: &str = include_str!("store/schema.sql");
32
33/// The oldest schema version a database is brought up from: 0.0.19's (ADR 0073).
34const OLDEST: u32 = 15;
35
36/// The steps from [`OLDEST`] on, in order, each with the version it brings a database to. A change to the schema
37/// adds one here and makes the same change to `schema.sql`, which a new database gets instead.
38const MIGRATIONS: &[(u32, &str)] = &[
39    (
40        16,
41        "ALTER TABLE jobs ADD COLUMN unseen INTEGER NOT NULL DEFAULT 0;",
42    ),
43    (
44        17,
45        "ALTER TABLE jobs ADD COLUMN cpu_periods INTEGER;\n\
46         ALTER TABLE jobs ADD COLUMN cpu_throttled INTEGER;\n\
47         ALTER TABLE jobs ADD COLUMN cpu_stalled INTEGER;\n\
48         ALTER TABLE samples ADD COLUMN cpu_periods INTEGER;\n\
49         ALTER TABLE samples ADD COLUMN cpu_throttled INTEGER;\n\
50         ALTER TABLE samples ADD COLUMN cpu_stalled INTEGER;",
51    ),
52    (
53        18,
54        "ALTER TABLE jobs ADD COLUMN slot_waited INTEGER NOT NULL DEFAULT 0;",
55    ),
56];
57
58/// The schema version this build reads and writes, the database's `user_version`: its last migration's.
59const VERSION: u32 = match MIGRATIONS.last() {
60    Some((version, _)) => *version,
61    None => OLDEST,
62};
63
64/// Bytes of log lines sent and not yet stored, at most: a job's capture waits beyond it, so a noisy job or a slow
65/// disk holds back that job's log, not the daemon's memory.
66pub(crate) const LOG_BACKLOG: u64 = 8 << 20;
67
68/// Host samples the writer holds before it stores them in one transaction: a minute's worth. One transaction per
69/// sample wrote about 300 MB a day to an idle daemon's WAL. A crash loses the samples held.
70const HOST_SAMPLE_BATCH: usize = 30;
71
72/// How many events a slow client may fall behind before its stream ends; it reconnects with `Last-Event-ID`.
73const EVENT_BUFFER: usize = 4096;
74
75/// Why the database cannot be used.
76#[derive(Debug, thiserror::Error)]
77pub enum StoreError {
78    /// SQLite failed.
79    #[error("{0}")]
80    Sqlite(#[from] rusqlite::Error),
81    /// A newer runnerd wrote the database.
82    #[error(
83        "the job history database is version {found}, newer than the {supported} this runnerd reads; \
84         run a newer runnerd or move the database aside"
85    )]
86    Newer {
87        /// The database's version.
88        found: u32,
89        /// The newest version this build knows.
90        supported: u32,
91    },
92    /// A runnerd before 0.0.19 wrote the database.
93    #[error(
94        "the job history database is version {found}, from a runnerd before 0.0.19; this runnerd reads versions \
95         from {oldest} on: move the database aside"
96    )]
97    Older {
98        /// The database's version.
99        found: u32,
100        /// The oldest version this build reads.
101        oldest: u32,
102    },
103    /// The directory or file cannot be created.
104    #[error("{0}")]
105    Io(#[from] std::io::Error),
106    /// A blocking read did not finish.
107    #[error("a database read stopped: {0}")]
108    Join(#[from] tokio::task::JoinError),
109    /// The database's `-wal` file holds changes not yet in it, and this user cannot open the database the way that
110    /// reads them.
111    #[error(
112        "{} holds changes not yet in the job history, which this user cannot read; run runnerd doctor as the user \
113         the daemon runs as",
114        .0.display()
115    )]
116    Unread(PathBuf),
117}
118
119/// A runner container as the history records it.
120#[derive(Debug, Clone, PartialEq, Eq)]
121pub struct RunnerRecord {
122    /// The runner's name, also its container's.
123    pub name: String,
124    /// `owner/name`.
125    pub repository: String,
126    /// The profile it runs in.
127    pub profile: String,
128    /// 1-based.
129    pub slot: usize,
130    /// The Docker container ID.
131    pub container: String,
132    /// When the container started, in milliseconds since the epoch.
133    pub started_at: u64,
134    /// The container's memory cap in bytes.
135    pub memory_cap: Option<u64>,
136    /// The container's CPU limit in millicores.
137    pub cpu_cap: Option<u64>,
138    /// How it was admitted; `None` for one a restart adopted without a record of it.
139    pub admission: Option<api::Admission>,
140}
141
142/// One sample of one runner's container.
143#[derive(Debug, Clone, PartialEq, Eq)]
144pub struct SampleRecord {
145    /// The runner.
146    pub runner: String,
147    /// When, in milliseconds since the epoch.
148    pub at: u64,
149    /// Millicores over the interval before it.
150    pub cpu: u64,
151    /// CPU time so far, in microseconds.
152    pub cpu_usage: u64,
153    /// Bytes without page cache (`observe::metrics::Usage::memory`).
154    pub memory: u64,
155    /// Bytes received since the container started.
156    pub network_received: u64,
157    /// Bytes sent since the container started.
158    pub network_sent: u64,
159    /// The container's `memory.events` so far.
160    pub events: Events,
161    /// What held the container's CPU back so far.
162    pub cpu_held: CpuHeld,
163}
164
165/// The last measurement of the cache volumes.
166#[derive(Debug, Clone, Default, PartialEq, Eq)]
167pub struct Measured {
168    /// When it was taken (seconds); none before the first measurement.
169    pub at: Option<u64>,
170    /// Each volume's size in bytes, by name.
171    pub bytes: BTreeMap<String, u64>,
172}
173
174/// The profiles and repositories as the stored events last announced them: what clients resuming after those
175/// events have.
176#[derive(Debug, Clone, Default, PartialEq, Eq)]
177pub struct Announced {
178    /// The last `profile.changed` of each profile not removed since, by name.
179    pub profiles: Vec<api::ProfileStatus>,
180    /// The last `repository.changed` of each repository not removed since, by name.
181    pub repositories: Vec<api::RepositoryStatus>,
182}
183
184/// A job the history has not closed, as the engine needs it after a restart.
185#[derive(Debug, Clone, PartialEq, Eq)]
186pub struct OpenJob {
187    /// GitHub's job ID.
188    pub id: u64,
189    /// `owner/name`.
190    pub repository: String,
191    /// `Queued` or `Running`.
192    pub state: api::JobState,
193    /// Its runner, once one took it.
194    pub runner: Option<String>,
195    /// When its runner was removed, in milliseconds since the epoch.
196    pub runner_removed_at: Option<u64>,
197    /// The workflow run it belongs to.
198    pub run_id: u64,
199    /// 1 for the run's first attempt.
200    pub run_attempt: u32,
201    /// The workflow's name, when known.
202    pub workflow: Option<String>,
203    /// When GitHub queued it, in milliseconds since the epoch.
204    pub queued_at: u64,
205    /// The labels it asked for, which tell its repository's pool.
206    pub labels: Vec<String>,
207}
208
209/// The job a slot ran last, as the history has it.
210#[derive(Debug, Clone, PartialEq, Eq)]
211pub struct SlotJob {
212    /// The profile the slot is in.
213    pub profile: String,
214    /// 1-based.
215    pub slot: usize,
216    /// The job, as admission tells jobs apart.
217    pub key: crate::admission::Key,
218}
219
220/// Where a job's log capture continues: the runner's page file and the byte offset in it.
221#[derive(Debug, Clone, Copy, PartialEq, Eq)]
222pub struct LogPosition {
223    /// The page number, from 1.
224    pub page: u64,
225    /// Bytes of that page already stored as lines.
226    pub offset: u64,
227}
228
229/// How far the mirror's log was counted: the time of the last response counted, in nanoseconds since the epoch as
230/// Docker logs it, and how many responses logged at exactly that time were counted, so the next read counts the
231/// ones it was too early to see.
232#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
233pub struct MirrorPosition {
234    /// Nanoseconds since the epoch.
235    pub at: u64,
236    /// Responses logged at `at` already counted.
237    pub seen: u64,
238}
239
240/// What the writer thread is asked to do, in order.
241#[derive(Debug)]
242enum Write {
243    Queued {
244        repository: String,
245        job: QueuedJob,
246    },
247    RunnerStarted(RunnerRecord),
248    RunnerAdmission {
249        runner: String,
250        admission: api::Admission,
251    },
252    RunnerOomKilled(String),
253    HostWaited {
254        jobs: Vec<u64>,
255        ms: u64,
256    },
257    SlotWaited {
258        jobs: Vec<u64>,
259        ms: u64,
260    },
261    BuildCache {
262        runner: String,
263        counts: api::BuildCacheCounts,
264    },
265    RunnerRemoved {
266        runner: String,
267        at: u64,
268    },
269    RunnersGone {
270        running: Vec<String>,
271        at: u64,
272    },
273    Claimed {
274        job: u64,
275        run_id: u64,
276        runner: String,
277        at: u64,
278    },
279    Details {
280        details: JobDetails,
281    },
282    Elsewhere {
283        details: JobDetails,
284    },
285    Lost {
286        job: u64,
287        at: u64,
288    },
289    Log {
290        job: u64,
291        lines: Vec<String>,
292        /// Where the capture resumes once these lines are stored; `None` for all but the last part of a batch.
293        position: Option<LogPosition>,
294        max_bytes: u64,
295        /// The lines' share of `LOG_BACKLOG`, given back once they are stored.
296        _backlog: OwnedSemaphorePermit,
297    },
298    Samples(Vec<SampleRecord>),
299    HostSample(api::HostSample),
300    MirrorBytes {
301        bytes: Vec<(String, u64)>,
302        until: MirrorPosition,
303        /// Whether the count committed.
304        done: oneshot::Sender<bool>,
305    },
306    CacheSizes {
307        at: u64,
308        sizes: BTreeMap<String, u64>,
309    },
310    /// Boxed: an event with a job's summary is many times the other writes' size.
311    Event(Box<Event>),
312    RecordMade(api::Record),
313    RecordSkipped {
314        job: u64,
315        why: String,
316    },
317    RecordRemoved {
318        job: u64,
319        at: u64,
320        by: api::RecordRemoval,
321    },
322    Paused {
323        repositories: Vec<String>,
324        /// Whether it committed.
325        done: oneshot::Sender<bool>,
326    },
327    Visibility {
328        repository: String,
329        visibility: api::Visibility,
330        at: u64,
331    },
332    SizingIntent {
333        change: api::SizingChange,
334        /// Its row, once it committed.
335        done: oneshot::Sender<Option<i64>>,
336    },
337    SizingResolved {
338        id: i64,
339        resolution: Resolution,
340        /// Whether it committed.
341        done: oneshot::Sender<bool>,
342    },
343    Notify {
344        change: NotifyChange,
345        /// Whether it committed.
346        done: oneshot::Sender<bool>,
347    },
348    Flush(oneshot::Sender<()>),
349}
350
351/// What the notifier records.
352#[derive(Debug)]
353enum NotifyChange {
354    /// Sets a target's keys and removes others.
355    Record {
356        target: String,
357        set: Vec<(String, u64)>,
358        remove: Vec<String>,
359    },
360    /// Forgets every target but these.
361    Keep(Vec<String>),
362}
363
364/// What became of a pending auto sizing change.
365#[derive(Debug, Clone, Copy, PartialEq, Eq)]
366pub enum Resolution {
367    /// The config file has it: announced as `sizing.changed`.
368    Done,
369    /// The file was not written.
370    Failed,
371    /// Left pending, and the config no longer says whether it was written: it keeps its interval and stays listed,
372    /// with what it replaced.
373    Uncertain,
374}
375
376/// Auto sizing as the history keeps it: when each profile last changed, and changes not yet known to be written.
377#[derive(Debug, Clone, Default, PartialEq, Eq)]
378pub struct AutoSizing {
379    /// The latest pending or done change of each profile, in seconds, however old.
380    pub last: BTreeMap<String, u64>,
381    /// Changes recorded before their file was written and not resolved since, by row.
382    pub pending: Vec<(i64, api::SizingChange)>,
383    /// Each profile's `cpus` as the daemon ran them, from its `profile.changed` events, oldest first: those `[history]
384    /// events` keeps, and each profile's latest however old.
385    pub cpus: Vec<sizing::CpusSet>,
386    /// Each profile's `memory` as the daemon ran it, from the same events.
387    pub memory: Vec<sizing::MemorySet>,
388    /// Each profile's slots as the daemon ran them, from the same events: one more for each slot a lowering left a
389    /// runner in until it ends.
390    pub slots: Vec<sizing::SlotsSet>,
391}
392
393/// The job history. Cloning shares it.
394#[derive(Debug, Clone)]
395pub struct Store {
396    writes: mpsc::UnboundedSender<Write>,
397    reader: Arc<Mutex<Connection>>,
398    /// Taken with `try_lock`, so one prune runs at a time; one that panicked midway left rows the next one deletes.
399    pruner: Arc<Mutex<Connection>>,
400    /// Set by `close`: a running prune stops after its batch, and none starts.
401    closed: Arc<AtomicBool>,
402    /// Host samples sent and not yet stored, oldest first; reads include them. The writer removes them once stored.
403    // Locked with `sync::lock`: every critical section is a push, a copy or a drain.
404    held: Arc<Mutex<Vec<api::HostSample>>>,
405    /// The last host sample sent, for admission to read without a query.
406    latest_host: Arc<Mutex<Option<api::HostSample>>>,
407    /// The last reading of the host's memory, which is not stored.
408    latest_host_memory: Arc<Mutex<Option<api::HostMemory>>>,
409    /// Each running runner's last sample, for admission to read without a query.
410    latest_samples: Arc<Mutex<BTreeMap<String, SampleRecord>>>,
411    events: broadcast::Sender<Arc<EventEnvelope>>,
412    backlog: Arc<Semaphore>,
413}
414
415impl Store {
416    /// Opens the database at `path`, creating it and its directory (mode 0700) if missing, with the schema when it
417    /// is new, and brings it to the schema this build reads. Refuses a database from before 0.0.19 or from a newer
418    /// runnerd. The time since the history's last sign of life is taken as time no runnerd ran: the jobs queued in
419    /// it leave it out of their waits.
420    pub fn open(path: &Path) -> Result<Store, StoreError> {
421        Store::open_at(path, crate::time::now_ms())
422    }
423
424    /// [`Store::open`] at `now` (ms).
425    fn open_at(path: &Path, now: u64) -> Result<Store, StoreError> {
426        if let Some(dir) = path.parent().filter(|dir| !dir.as_os_str().is_empty()) {
427            std::fs::DirBuilder::new()
428                .recursive(true)
429                .mode(0o700)
430                .create(dir)?;
431        }
432        // Owner-only, as SQLite then makes its `-wal` and `-shm` files: logs may say more than the owner wants shared.
433        std::fs::OpenOptions::new()
434            .create(true)
435            .append(true)
436            .mode(0o600)
437            .open(path)?;
438        let mut db = Connection::open(path)?;
439        configure(&db)?;
440        prepare(&mut db)?;
441        let downtime = Downtime::before(&mut db, now)?;
442        let reader = Connection::open_with_flags(
443            path,
444            OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX,
445        )?;
446        configure(&reader)?;
447        let pruner = Connection::open(path)?;
448        configure(&pruner)?;
449        let (events, _) = broadcast::channel(EVENT_BUFFER);
450        let (writes, receiver) = mpsc::unbounded_channel();
451        let thread_events = events.clone();
452        let held = Arc::new(Mutex::new(Vec::new()));
453        let thread_held = held.clone();
454        std::thread::Builder::new()
455            .name("runnerd-store".to_owned())
456            .spawn(move || write::run(db, downtime, receiver, &thread_events, &thread_held))?;
457        Ok(Store {
458            writes,
459            reader: Arc::new(Mutex::new(reader)),
460            pruner: Arc::new(Mutex::new(pruner)),
461            closed: Arc::new(AtomicBool::new(false)),
462            held,
463            latest_host: Arc::new(Mutex::new(None)),
464            latest_host_memory: Arc::new(Mutex::new(None)),
465            latest_samples: Arc::new(Mutex::new(BTreeMap::new())),
466            events,
467            backlog: Arc::new(Semaphore::new(
468                usize::try_from(LOG_BACKLOG).unwrap_or(usize::MAX),
469            )),
470        })
471    }
472
473    fn send(&self, write: Write) {
474        // Fails only once the writer thread is gone, after a SQLite error it logged.
475        let _ = self.writes.send(write);
476    }
477
478    /// Records a queued job the repository's runners may run; announces it the first time.
479    pub fn queued(&self, repository: &str, job: &QueuedJob) {
480        self.send(Write::Queued {
481            repository: repository.to_owned(),
482            job: job.clone(),
483        });
484    }
485
486    /// Records a runner container that started, or one an earlier daemon started and this one adopted.
487    pub fn runner_started(&self, runner: RunnerRecord) {
488        self.send(Write::RunnerStarted(runner));
489    }
490
491    /// Records that a runner's container and registration were removed at `at` (ms).
492    pub fn runner_removed(&self, runner: &str, at: u64) {
493        lock(&self.latest_samples).remove(runner);
494        self.send(Write::RunnerRemoved {
495            runner: runner.to_owned(),
496            at,
497        });
498    }
499
500    /// Records every runner the history holds open, except the `running` ones, removed at `at` (ms): what recovery
501    /// found gone, so it stops counting as holding a slot.
502    pub fn runners_gone(&self, running: Vec<String>, at: u64) {
503        lock(&self.latest_samples).retain(|runner, _| running.contains(runner));
504        self.send(Write::RunnersGone { running, at });
505    }
506
507    /// Records that a runner took a job at `at` (ms).
508    pub fn claimed(&self, job: u64, run_id: u64, runner: &str, at: u64) {
509        self.send(Write::Claimed {
510            job,
511            run_id,
512            runner: runner.to_owned(),
513            at,
514        });
515    }
516
517    /// Records a running job's progress and steps as GitHub reports them; announces changed steps.
518    pub fn details(&self, details: JobDetails) {
519        self.send(Write::Details { details });
520    }
521
522    /// Closes a queued job that left the queue without a runner of this daemon: another machine's runner took it, or
523    /// none did and it has no start.
524    pub fn elsewhere(&self, details: JobDetails) {
525        self.send(Write::Elsewhere { details });
526    }
527
528    /// Closes a job whose container is gone and that GitHub did not report completed in time.
529    pub fn lost(&self, job: u64, at: u64) {
530        self.send(Write::Lost { job, at });
531    }
532
533    /// Appends log lines to a job, up to `max_bytes` per job, and records where the capture continues. Waits while
534    /// `LOG_BACKLOG` bytes of lines are waiting to be stored; a larger batch goes in parts, each waiting its turn.
535    pub async fn log(&self, job: u64, lines: Vec<String>, position: LogPosition, max_bytes: u64) {
536        let size = |line: &String| line.len() as u64 + 1;
537        let mut lines = lines.into_iter().map(|mut line| {
538            // No line takes the whole backlog; the capture already cuts lines far shorter.
539            if size(&line) > LOG_BACKLOG {
540                let mut end = usize::try_from(LOG_BACKLOG / 2).unwrap_or(usize::MAX);
541                while !line.is_char_boundary(end) {
542                    end -= 1;
543                }
544                line.truncate(end);
545            }
546            line
547        });
548        let mut next = lines.next();
549        loop {
550            let (mut part, mut bytes) = (Vec::new(), 0);
551            while let Some(line) = next.take_if(|line| bytes + size(line) <= LOG_BACKLOG) {
552                bytes += size(&line);
553                part.push(line);
554                next = lines.next();
555            }
556            let last = next.is_none();
557            let share = u32::try_from(bytes.max(1)).unwrap_or(u32::MAX);
558            // The semaphore is never closed.
559            let Ok(backlog) = self.backlog.clone().acquire_many_owned(share).await else {
560                return;
561            };
562            self.send(Write::Log {
563                job,
564                lines: part,
565                position: last.then_some(position),
566                max_bytes,
567                _backlog: backlog,
568            });
569            if last {
570                return;
571            }
572        }
573    }
574
575    /// Records samples of runner containers.
576    pub fn samples(&self, samples: Vec<SampleRecord>) {
577        let mut latest = lock(&self.latest_samples);
578        for sample in &samples {
579            latest.insert(sample.runner.clone(), sample.clone());
580        }
581        drop(latest);
582        self.send(Write::Samples(samples));
583    }
584
585    /// Records a sample of the slice and sends it to every subscriber at once; it is stored with the next ones, a
586    /// minute's worth in one transaction, or at `flush`.
587    pub fn host_sample(&self, sample: api::HostSample) {
588        *lock(&self.latest_host) = Some(sample.clone());
589        lock(&self.held).push(sample.clone());
590        self.send(Write::HostSample(sample));
591    }
592
593    /// A runner's last sample, stored or not; `None` before its first sample or once it is removed.
594    pub fn latest_sample(&self, runner: &str) -> Option<SampleRecord> {
595        lock(&self.latest_samples).get(runner).cloned()
596    }
597
598    /// Records what a runner reserves now, when measured admission changed it after the start.
599    pub fn runner_admission(&self, runner: &str, admission: api::Admission) {
600        self.send(Write::RunnerAdmission {
601            runner: runner.to_owned(),
602            admission,
603        });
604    }
605
606    /// Records that Docker saw the kernel kill a process in a runner's container for memory, which its samples may
607    /// have missed: one in the last interval before the container stopped.
608    pub fn runner_oom_killed(&self, runner: &str) {
609        self.send(Write::RunnerOomKilled(runner.to_owned()));
610    }
611
612    /// Adds what the shared build cache served a runner and took from it, and records whether its job could write.
613    pub fn build_cache(&self, runner: &str, counts: api::BuildCacheCounts) {
614        self.send(Write::BuildCache {
615            runner: runner.to_owned(),
616            counts,
617        });
618    }
619
620    /// Keeps the latest reading of the host's memory for admission and the status; it is not stored.
621    pub fn host_memory(&self, memory: api::HostMemory) {
622        *lock(&self.latest_host_memory) = Some(memory);
623    }
624
625    /// The latest reading of the host's memory; `None` before the first.
626    pub fn latest_host_memory(&self) -> Option<api::HostMemory> {
627        lock(&self.latest_host_memory).clone()
628    }
629
630    /// Adds `ms` to how long each of `jobs` waited because the host's memory could not take its runner, which
631    /// sizing does not read as a wait for a slot.
632    pub fn host_waited(&self, jobs: Vec<u64>, ms: u64) {
633        self.send(Write::HostWaited { jobs, ms });
634    }
635
636    /// Adds `ms` to how long each of `jobs` waited with no slot of its profile free while the budgets had room for
637    /// its runner: what auto sizing reads as a wait for a slot (ADR 0090).
638    pub fn slot_waited(&self, jobs: Vec<u64>, ms: u64) {
639        self.send(Write::SlotWaited { jobs, ms });
640    }
641
642    /// The last host sample sent, stored or not; `None` before the first.
643    pub fn latest_host_sample(&self) -> Option<api::HostSample> {
644        lock(&self.latest_host).clone()
645    }
646
647    /// Adds bytes the mirror served to runners, counted from its log up to `until`, and says whether that was
648    /// stored: if not, the caller counts the same lines again.
649    pub async fn mirror_bytes(&self, bytes: Vec<(String, u64)>, until: MirrorPosition) -> bool {
650        let (done, committed) = oneshot::channel();
651        self.send(Write::MirrorBytes { bytes, until, done });
652        committed.await.unwrap_or(false)
653    }
654
655    /// Records volume sizes in bytes, by name, measured at `at` (ms).
656    pub fn cache_sizes(&self, at: u64, sizes: BTreeMap<String, u64>) {
657        self.send(Write::CacheSizes { at, sizes });
658    }
659
660    /// Stores a durable event, or sends a live one, to every subscriber.
661    pub fn event(&self, event: Event) {
662        self.send(Write::Event(Box::new(event)));
663    }
664
665    /// Records that a failed job's runner was kept as `record`, and announces it.
666    pub fn record_made(&self, record: api::Record) {
667        self.send(Write::RecordMade(record));
668    }
669
670    /// Records why a failed job's runner was not kept.
671    pub fn record_skipped(&self, job: u64, why: String) {
672        self.send(Write::RecordSkipped { job, why });
673    }
674
675    /// Records that a job's record was removed at `at` (ms), and why, and announces it.
676    pub fn record_removed(&self, job: u64, at: u64, by: api::RecordRemoval) {
677        self.send(Write::RecordRemoved { job, at, by });
678    }
679
680    /// Records the paused repositories, all of them, in place of the ones recorded before. The write is sent at
681    /// once, in order with every other; the future says whether it committed.
682    pub fn paused(&self, repositories: Vec<String>) -> impl Future<Output = bool> + use<> {
683        let (done, committed) = oneshot::channel();
684        self.send(Write::Paused { repositories, done });
685        async move { committed.await.unwrap_or(false) }
686    }
687
688    /// Records a repository's visibility as GitHub gave it at `at` (ms), in place of the one recorded before.
689    pub fn visibility(&self, repository: String, visibility: api::Visibility, at: u64) {
690        self.send(Write::Visibility {
691            repository,
692            visibility,
693            at,
694        });
695    }
696
697    /// Records a change auto sizing is about to write, as pending, before the file is, synced to disk. The future
698    /// has its row once it committed, `None` if it did not: then nothing may be written.
699    pub fn sizing_intent(
700        &self,
701        change: api::SizingChange,
702    ) -> impl Future<Output = Option<i64>> + use<> {
703        let (done, committed) = oneshot::channel();
704        self.send(Write::SizingIntent { change, done });
705        async move { committed.await.ok().flatten() }
706    }
707
708    /// Records what became of a pending change, synced to disk; a done one is announced as `sizing.changed` in the
709    /// same transaction. The future says whether it committed.
710    pub fn sizing_resolved(
711        &self,
712        id: i64,
713        resolution: Resolution,
714    ) -> impl Future<Output = bool> + use<> {
715        let (done, committed) = oneshot::channel();
716        self.send(Write::SizingResolved {
717            id,
718            resolution,
719            done,
720        });
721        async move { committed.await.unwrap_or(false) }
722    }
723
724    /// Records what the notifier delivered to a target: sets `set`'s keys and removes `remove`'s, in one
725    /// transaction. The future says whether it committed.
726    pub fn notify_record(
727        &self,
728        target: &str,
729        set: Vec<(String, u64)>,
730        remove: Vec<String>,
731    ) -> impl Future<Output = bool> + use<> {
732        let (done, committed) = oneshot::channel();
733        self.send(Write::Notify {
734            change: NotifyChange::Record {
735                target: target.to_owned(),
736                set,
737                remove,
738            },
739            done,
740        });
741        async move { committed.await.unwrap_or(false) }
742    }
743
744    /// Forgets what the notifier kept for every target but `targets`. The future says whether it committed.
745    pub fn notify_keep(&self, targets: Vec<String>) -> impl Future<Output = bool> + use<> {
746        let (done, committed) = oneshot::channel();
747        self.send(Write::Notify {
748            change: NotifyChange::Keep(targets),
749            done,
750        });
751        async move { committed.await.unwrap_or(false) }
752    }
753
754    /// Deletes what is older than `history` keeps, at `now` (ms), off the writer, in batches the writer's
755    /// transactions fit between. A prune while the last one runs does nothing. The handle finishes with it; dropping
756    /// it lets it run.
757    pub fn prune(&self, now: u64, history: &History) -> tokio::task::JoinHandle<()> {
758        let pruner = self.pruner.clone();
759        let closed = self.closed.clone();
760        let history = history.clone();
761        tokio::task::spawn_blocking(move || {
762            let db = match pruner.try_lock() {
763                Ok(db) => db,
764                Err(TryLockError::Poisoned(poisoned)) => poisoned.into_inner(),
765                Err(TryLockError::WouldBlock) => return,
766            };
767            let db = prune::Db {
768                connection: &db,
769                stop: &closed,
770            };
771            if !db.stop.load(Ordering::Relaxed) {
772                prune::run(&db, now, &history);
773            }
774        })
775    }
776
777    /// Stops a running prune after its batch, and any later one before it starts, so a shutdown does not wait for
778    /// one. What a stopped prune left, the next daemon's deletes. Writes go on as before.
779    pub fn close(&self) {
780        self.closed.store(true, Ordering::Relaxed);
781    }
782
783    /// Waits until every write sent before is done.
784    pub async fn flush(&self) {
785        let (done, wait) = oneshot::channel();
786        self.send(Write::Flush(done));
787        let _ = wait.await;
788    }
789
790    /// Every event from now on, durable or live. Subscribe before reading what was stored, then skip what the read
791    /// returned, and nothing is missed or sent twice.
792    pub fn subscribe(&self) -> broadcast::Receiver<Arc<EventEnvelope>> {
793        self.events.subscribe()
794    }
795
796    async fn read<T: Send + 'static>(
797        &self,
798        query: impl FnOnce(&Connection) -> rusqlite::Result<T> + Send + 'static,
799    ) -> Result<T, StoreError> {
800        let reader = self.reader.clone();
801        // A panic during a read leaves the connection usable: reads change nothing.
802        Ok(tokio::task::spawn_blocking(move || query(&lock(&reader))).await??)
803    }
804
805    /// One page of jobs, newest first.
806    pub async fn jobs(&self, query: api::JobQuery) -> Result<api::JobPage, StoreError> {
807        self.read(move |db| read::jobs(db, &query)).await
808    }
809
810    /// A job with its steps.
811    pub async fn job(&self, id: u64) -> Result<Option<api::Job>, StoreError> {
812        self.read(move |db| read::job(db, id)).await
813    }
814
815    /// A job's samples, oldest first; `None` if there is no such job.
816    pub async fn job_samples(
817        &self,
818        id: u64,
819        interval: Duration,
820    ) -> Result<Option<api::SampleSeries>, StoreError> {
821        self.read(move |db| read::job_samples(db, id, interval))
822            .await
823    }
824
825    /// Up to `limit` log lines of a job from line `from`, with their numbers.
826    pub async fn log_lines(
827        &self,
828        job: u64,
829        from: u64,
830        limit: u64,
831    ) -> Result<Vec<(u64, String)>, StoreError> {
832        self.read(move |db| read::log_lines(db, job, from, limit))
833            .await
834    }
835
836    /// Up to `limit` durable events after `after`, oldest first.
837    pub async fn events_after(
838        &self,
839        after: u64,
840        limit: u64,
841    ) -> Result<Vec<EventEnvelope>, StoreError> {
842        self.read(move |db| read::events_after(db, after, limit))
843            .await
844    }
845
846    /// The changes auto sizing wrote since `since` (ms), oldest first.
847    pub async fn sizing_changes(&self, since: u64) -> Result<Vec<api::SizingChange>, StoreError> {
848        self.read(move |db| read::sizing_changes(db, since)).await
849    }
850
851    /// When each profile last changed, and the changes still pending.
852    pub async fn auto_sizing(&self) -> Result<AutoSizing, StoreError> {
853        self.read(read::auto_sizing).await
854    }
855
856    /// The profiles and repositories as the stored events last announced them, so a restarted daemon announces
857    /// only what changed since, including what its config no longer has. Blocks on the database: for startup.
858    pub fn announced(&self) -> Result<Announced, StoreError> {
859        Ok(read::announced(&lock(&self.reader))?)
860    }
861
862    /// The repositories recorded paused, for the engine to start with. Blocks on the database: for startup.
863    pub fn paused_repositories(&self) -> Result<Vec<String>, StoreError> {
864        Ok(read::paused(&lock(&self.reader))?)
865    }
866
867    /// The slice's samples since `since` (seconds), oldest first, the ones not yet stored included.
868    pub async fn host_samples(&self, since: u64) -> Result<Vec<api::HostSample>, StoreError> {
869        // Copied before the read: the writer removes a sample only once it is stored, so each one is in the copy,
870        // the read, or both.
871        let held = lock(&self.held).clone();
872        let mut samples = self.read(move |db| read::host_samples(db, since)).await?;
873        let stored = samples.last().map_or(0, |sample| sample.at);
874        samples.extend(
875            held.into_iter()
876                .filter(|sample| sample.at >= since && sample.at > stored),
877        );
878        Ok(samples)
879    }
880
881    /// The jobs runnerd's runners started and finished from `since` until `now` (seconds).
882    pub async fn activity(&self, since: u64, now: u64) -> Result<api::Activity, StoreError> {
883        self.read(move |db| read::activity(db, since, now)).await
884    }
885
886    /// Finished jobs per repository and bucket.
887    pub async fn stats(&self, query: api::StatsQuery, now: u64) -> Result<api::Stats, StoreError> {
888        self.read(move |db| read::stats(db, &query, now)).await
889    }
890
891    /// What sizing reads: the finished jobs queued at or after `since` (ms) and the runners since.
892    pub async fn sizing(&self, since: u64) -> Result<sizing::History, StoreError> {
893        self.read(move |db| read::sizing(db, since)).await
894    }
895
896    /// What pricing at GitHub's rates reads: the completed and lost jobs dated at or after `since` (ms), each
897    /// repository's visibility and since when the history holds every final job.
898    pub async fn pricing(&self, since: u64) -> Result<api::PricingHistory, StoreError> {
899        self.read(move |db| read::pricing(db, since)).await
900    }
901
902    /// The jobs completed or lost that were queued at or after `since` (ms), as insights read them, and from when the
903    /// history holds every job.
904    pub async fn insights(&self, since: u64) -> Result<crate::insights::History, StoreError> {
905        self.read(move |db| read::insights(db, since)).await
906    }
907
908    /// The newest durable event's ID; 0 before the first.
909    pub async fn last_event(&self) -> Result<u64, StoreError> {
910        self.read(read::last_event).await
911    }
912
913    /// What the notifier kept for each target, by target, then key.
914    pub async fn notify_state(
915        &self,
916    ) -> Result<BTreeMap<String, BTreeMap<String, u64>>, StoreError> {
917        self.read(read::notify_state).await
918    }
919
920    /// The jobs queued at or after `since` (ms), whatever became of them, as warm runners learn from them.
921    pub async fn warm(&self, since: u64) -> Result<Vec<warm::Seen>, StoreError> {
922        self.read(move |db| read::warm(db, since)).await
923    }
924
925    /// The last measurement of every volume, with its time (seconds), by name.
926    pub async fn cache_sizes_now(&self) -> Result<Measured, StoreError> {
927        self.read(read::latest_cache_sizes).await
928    }
929
930    /// What the build cache served and took from the runners started since `since` (ms), summed.
931    pub async fn build_cache_since(&self, since: u64) -> Result<api::BuildCacheCounts, StoreError> {
932        self.read(move |db| read::build_cache_since(db, since))
933            .await
934    }
935
936    /// Every volume measurement since `since` (seconds).
937    pub async fn cache_history(&self, since: u64) -> Result<Vec<api::CacheSize>, StoreError> {
938        self.read(move |db| read::cache_history(db, since)).await
939    }
940
941    /// The jobs not yet closed, for the engine after a restart.
942    pub async fn open_jobs(&self) -> Result<Vec<OpenJob>, StoreError> {
943        self.read(read::open_jobs).await
944    }
945
946    /// The newest named job each slot ran, for the engine after a restart.
947    pub async fn slot_jobs(&self) -> Result<Vec<SlotJob>, StoreError> {
948        self.read(read::slot_jobs).await
949    }
950
951    /// The job a runner took, if the history knows it.
952    pub async fn runner_job(&self, runner: &str) -> Result<Option<u64>, StoreError> {
953        let runner = runner.to_owned();
954        self.read(move |db| read::runner_job(db, &runner)).await
955    }
956
957    /// How a runner was admitted, with what it reserves since, if the history knows it.
958    pub async fn runner_admitted(
959        &self,
960        runner: &str,
961    ) -> Result<Option<api::Admission>, StoreError> {
962        let runner = runner.to_owned();
963        self.read(move |db| read::runner_admission(db, &runner))
964            .await
965    }
966
967    /// Where a job's log capture continues.
968    pub async fn log_position(&self, job: u64) -> Result<LogPosition, StoreError> {
969        self.read(move |db| read::log_position(db, job)).await
970    }
971
972    /// How far the mirror's log was counted.
973    pub async fn mirror_log_until(&self) -> Result<MirrorPosition, StoreError> {
974        self.read(read::mirror_log_until).await
975    }
976}
977
978/// What sizing reads from the database at `path`, opened read-only and left as it is, as `runnerd doctor` reads it
979/// while no daemon runs; `None` when there is no database yet. In a directory it may write, SQLite keeps the
980/// `-wal` and `-shm` files a read of a WAL database needs next to it; in one it may not, it reads the file as it is,
981/// or refuses when the `-wal` holds changes the file does not have yet.
982pub fn sizing_at(path: &Path, since: u64) -> Result<Option<sizing::History>, StoreError> {
983    if !path.exists() {
984        return Ok(None);
985    }
986    let read = |path: &str, flags| -> Result<sizing::History, StoreError> {
987        let db = Connection::open_with_flags(path, flags)?;
988        db.busy_timeout(Duration::from_secs(5))?;
989        sizing_of(&db, since)
990    };
991    let name = path.to_string_lossy();
992    match read(&name, OpenFlags::SQLITE_OPEN_READ_ONLY) {
993        Err(StoreError::Sqlite(error))
994            if matches!(
995                error.sqlite_error_code(),
996                Some(rusqlite::ErrorCode::ReadOnly | rusqlite::ErrorCode::CannotOpen)
997            ) =>
998        {
999            // No `-shm` can be made: read the file alone, unless the `-wal` holds what it lacks.
1000            let mut wal = path.as_os_str().to_owned();
1001            wal.push("-wal");
1002            let wal = PathBuf::from(wal);
1003            if std::fs::metadata(&wal).is_ok_and(|wal| wal.len() > 0) {
1004                return Err(StoreError::Unread(wal));
1005            }
1006            let uri = format!(
1007                "file:{}?mode=ro&immutable=1",
1008                name.replace('%', "%25")
1009                    .replace('?', "%3f")
1010                    .replace('#', "%23")
1011            );
1012            read(
1013                &uri,
1014                OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_URI,
1015            )
1016        }
1017        history => history,
1018    }
1019    .map(Some)
1020}
1021
1022/// What sizing reads from the database, without creating its schema; an older one reads as [`VERSION`].
1023fn sizing_of(db: &Connection, since: u64) -> Result<sizing::History, StoreError> {
1024    match version(db)? {
1025        0 => return Ok(sizing::History::default()),
1026        VERSION => {}
1027        _ => read_as_current(db)?,
1028    }
1029    let mut history = read::sizing(db, since)?;
1030    let auto = read::auto_sizing(db)?;
1031    history.memory_set = auto.memory;
1032    history.slots_set = auto.slots;
1033    history.changes = read::sizing_changes(db, since.saturating_sub(sizing::memory::HISTORY))?;
1034    Ok(history)
1035}
1036
1037/// The database's schema version: 0 for a new one, else from [`OLDEST`] to [`VERSION`]; any other is refused.
1038fn version(db: &Connection) -> Result<u32, StoreError> {
1039    let found: u32 = db.pragma_query_value(None, "user_version", |row| row.get(0))?;
1040    match found {
1041        0 | OLDEST..=VERSION => Ok(found),
1042        found if found > VERSION => Err(StoreError::Newer {
1043            found,
1044            supported: VERSION,
1045        }),
1046        found => Err(StoreError::Older {
1047            found,
1048            oldest: OLDEST,
1049        }),
1050    }
1051}
1052
1053/// Lets a read-only look at a database from before [`VERSION`], which it cannot migrate, read it as this build
1054/// does: a table it lacks becomes an empty temporary one, and a table that lacks columns a temporary view of it
1055/// that adds them, each with its default. Temporary objects come first in name lookups.
1056fn read_as_current(db: &Connection) -> Result<(), StoreError> {
1057    let current = Connection::open_in_memory()?;
1058    current.execute_batch(SCHEMA)?;
1059    let tables: Vec<(String, String)> = current
1060        .prepare(
1061            "SELECT name, sql FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'",
1062        )?
1063        .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?
1064        .collect::<rusqlite::Result<_>>()?;
1065    let columns =
1066        |db: &Connection, table: &str| -> rusqlite::Result<Vec<(String, Option<String>)>> {
1067            db.prepare("SELECT name, dflt_value FROM pragma_table_info(?1, 'main')")?
1068                .query_map([table], |row| Ok((row.get(0)?, row.get(1)?)))?
1069                .collect()
1070        };
1071    for (table, sql) in tables {
1072        let found: Vec<String> = columns(db, &table)?
1073            .into_iter()
1074            .map(|(name, _)| name)
1075            .collect();
1076        if found.is_empty() {
1077            db.execute_batch(&sql.replacen("CREATE TABLE", "CREATE TEMP TABLE", 1))?;
1078            continue;
1079        }
1080        let added: Vec<String> = columns(&current, &table)?
1081            .into_iter()
1082            .filter(|(name, _)| !found.contains(name))
1083            .map(|(name, default)| format!("{} AS {name}", default.as_deref().unwrap_or("NULL")))
1084            .collect();
1085        if !added.is_empty() {
1086            db.execute_batch(&format!(
1087                "CREATE TEMP VIEW {table} AS SELECT *, {} FROM main.{table}",
1088                added.join(", ")
1089            ))?;
1090        }
1091    }
1092    Ok(())
1093}
1094
1095/// WAL, so the API reads while the writer writes; a busy timeout instead of an immediate error.
1096fn configure(db: &Connection) -> rusqlite::Result<()> {
1097    db.pragma_update(None, "journal_mode", "wal")?;
1098    db.pragma_update(None, "synchronous", "normal")?;
1099    db.pragma_update(None, "foreign_keys", "on")?;
1100    db.busy_timeout(Duration::from_secs(5))
1101}
1102
1103/// Creates the schema in a new database, in one transaction with its version, or runs the migrations an older one
1104/// lacks, each in its own transaction with the version it brings it to.
1105fn prepare(db: &mut Connection) -> Result<(), StoreError> {
1106    let found = version(db)?;
1107    if found == 0 {
1108        let transaction = db.transaction()?;
1109        transaction.execute_batch(SCHEMA)?;
1110        transaction.pragma_update(None, "user_version", VERSION)?;
1111        transaction.commit()?;
1112        return Ok(());
1113    }
1114    for (version, migration) in MIGRATIONS.iter().filter(|(version, _)| *version > found) {
1115        let transaction = db.transaction()?;
1116        transaction.execute_batch(migration)?;
1117        transaction.pragma_update(None, "user_version", version)?;
1118        transaction.commit()?;
1119    }
1120    Ok(())
1121}
1122
1123/// The time before this daemon when no runnerd ran, in ms: from the history's last sign of life, its newest host
1124/// sample, event or open, to this daemon's start. Host samples are stored a minute's worth at a time, so a crash
1125/// may count up to a minute more. Without a sign of life, as in a new history, there is none: the history's start
1126/// clips the waits of jobs queued before it (`history_kept`).
1127#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1128struct Downtime {
1129    from: u64,
1130    until: u64,
1131}
1132
1133impl Downtime {
1134    /// The downtime before a daemon that starts at `now`; the jobs still queued in the history leave it out of
1135    /// their waits, as the ones first seen after do when they are recorded. The open is a sign of life itself, kept
1136    /// with them: a start that fails after it, and is retried, does not hand out the same downtime again.
1137    fn before(db: &mut Connection, now: u64) -> Result<Option<Downtime>, StoreError> {
1138        // Immediate: it reads, then writes. A deferred transaction that has read cannot wait for the write lock, and
1139        // fails at once while another connection, such as a dropped store's writer finishing, holds it.
1140        let transaction = db.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?;
1141        let last: Option<u64> = transaction.query_row(
1142            "SELECT max(at) FROM (SELECT max(at) AS at FROM host_samples UNION ALL SELECT max(at) FROM events \
1143             UNION ALL SELECT value FROM meta WHERE key = 'opened_at')",
1144            [],
1145            |row| row.get(0),
1146        )?;
1147        let downtime = last
1148            .filter(|last| *last < now)
1149            .map(|from| Downtime { from, until: now });
1150        if let Some(downtime) = downtime {
1151            let queued: Vec<(u64, u64)> = transaction
1152                .prepare("SELECT id, queued_at FROM jobs WHERE state = 'queued'")?
1153                .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?
1154                .collect::<rusqlite::Result<_>>()?;
1155            for (job, queued_at) in queued {
1156                transaction.execute(
1157                    "UPDATE jobs SET unseen = unseen + ?2 WHERE id = ?1",
1158                    rusqlite::params![job, downtime.of(queued_at)],
1159                )?;
1160            }
1161        }
1162        transaction.execute(
1163            "INSERT OR REPLACE INTO meta (key, value) VALUES ('opened_at', ?1)",
1164            [now],
1165        )?;
1166        transaction.commit()?;
1167        Ok(downtime)
1168    }
1169
1170    /// How much of it a job queued at `queued_at` (ms) waited through.
1171    fn of(self, queued_at: u64) -> u64 {
1172        self.until.saturating_sub(queued_at.max(self.from))
1173    }
1174
1175    /// How much of what [`Downtime::of`] gave a job queued at `queued_at` came after it started at `started_at`
1176    /// (ms), as for a job a runner that outlived the last daemon took while none ran.
1177    fn after(self, queued_at: u64, started_at: u64) -> u64 {
1178        self.of(queued_at)
1179            .saturating_sub(started_at.saturating_sub(queued_at.max(self.from)))
1180    }
1181}
1182
1183/// Logs a failed write; the writer goes on with the next one.
1184fn logged<T>(what: &str, result: rusqlite::Result<T>) -> Option<T> {
1185    result
1186        .map_err(|error| warn!("job history: cannot {what}: {error}"))
1187        .ok()
1188}
1189
1190#[cfg(test)]
1191mod tests;