Skip to main content

runnerd_core/
store.rs

1//! The job history: a SQLite database of jobs, their runners, steps, samples and logs, the events clients resume
2//! from, cache sizes, the paused repositories and where the notifier got to. Owns the schema, the one writer and
3//! the event broadcast. Writes are messages to a writer thread, so the engine never waits on the disk; reads and
4//! pruning run on Tokio's blocking pool, pruning on its own connection, a batch at a time. It decides nothing about
5//! scheduling and stores no secret: no token, no JIT config, no runner `_diag` log.
6
7use std::collections::{BTreeMap, BTreeSet};
8use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt};
9use std::path::{Path, PathBuf};
10use std::sync::atomic::{AtomicBool, Ordering};
11use std::sync::{Arc, Mutex, TryLockError};
12use std::time::Duration;
13
14use rusqlite::{Connection, OpenFlags};
15use tokio::sync::{OwnedSemaphorePermit, Semaphore, broadcast, mpsc, oneshot};
16use tracing::warn;
17
18use crate::admission::trace::Trace;
19use crate::api::{self, Event, EventEnvelope};
20use crate::config::History;
21use crate::github::{JobDetails, NamedRun, QueuedJob};
22use crate::observe::disk::{Window, Written};
23use crate::observe::metrics::{CpuHeld, Events};
24use crate::scheduler::warm;
25use crate::sizing;
26use crate::sync::lock;
27
28mod lens;
29mod outline;
30mod prune;
31mod read;
32mod run;
33mod write;
34
35pub use run::{Claim, RunJobs, RunnerFault};
36
37/// The schema a new database is created with, at [`VERSION`].
38const SCHEMA: &str = include_str!("store/schema.sql");
39
40/// The oldest schema version a database is brought up from: 0.0.19's (ADR 0073).
41const OLDEST: u32 = 15;
42
43/// The steps from [`OLDEST`] on, in order, each with the version it brings a database to. A change to the schema
44/// adds one here and makes the same change to `schema.sql`, which a new database gets instead.
45const MIGRATIONS: &[(u32, &str)] = &[
46    (
47        16,
48        "ALTER TABLE jobs ADD COLUMN unseen INTEGER NOT NULL DEFAULT 0;",
49    ),
50    (
51        17,
52        "ALTER TABLE jobs ADD COLUMN cpu_periods INTEGER;\n\
53         ALTER TABLE jobs ADD COLUMN cpu_throttled INTEGER;\n\
54         ALTER TABLE jobs ADD COLUMN cpu_stalled INTEGER;\n\
55         ALTER TABLE samples ADD COLUMN cpu_periods INTEGER;\n\
56         ALTER TABLE samples ADD COLUMN cpu_throttled INTEGER;\n\
57         ALTER TABLE samples ADD COLUMN cpu_stalled INTEGER;",
58    ),
59    (
60        18,
61        "ALTER TABLE jobs ADD COLUMN slot_waited INTEGER NOT NULL DEFAULT 0;",
62    ),
63    (
64        19,
65        "ALTER TABLE jobs ADD COLUMN seen_at INTEGER;\n\
66         ALTER TABLE runners ADD COLUMN purpose TEXT;\n\
67         ALTER TABLE runners ADD COLUMN stopped_for TEXT;",
68    ),
69    (
70        20,
71        "ALTER TABLE jobs ADD COLUMN runner_fault INTEGER NOT NULL DEFAULT 0;",
72    ),
73    (21, "ALTER TABLE runners ADD COLUMN cancel_by INTEGER;"),
74    (
75        22,
76        "CREATE TABLE log_outlines (job INTEGER PRIMARY KEY NOT NULL, from_line INTEGER NOT NULL, \
77         errors INTEGER NOT NULL, warnings INTEGER NOT NULL, groups_kept INTEGER NOT NULL, \
78         marks_kept INTEGER NOT NULL, sections_kept INTEGER NOT NULL, dropped INTEGER NOT NULL, \
79         hidden INTEGER NOT NULL);\n\
80         CREATE TABLE log_groups (job INTEGER NOT NULL, start INTEGER NOT NULL, end_line INTEGER, \
81         depth INTEGER NOT NULL, title TEXT NOT NULL, PRIMARY KEY (job, start)) WITHOUT ROWID;\n\
82         CREATE INDEX log_groups_open ON log_groups (job, start) WHERE end_line IS NULL;\n\
83         CREATE TABLE log_marks (job INTEGER NOT NULL, line INTEGER NOT NULL, kind TEXT NOT NULL, \
84         PRIMARY KEY (job, line)) WITHOUT ROWID;",
85    ),
86    (
87        23,
88        "CREATE TABLE sessions (id INTEGER PRIMARY KEY, started_at INTEGER NOT NULL, sampled_at INTEGER);\n\
89         ALTER TABLE records ADD COLUMN gone_at INTEGER;",
90    ),
91    (
92        24,
93        "ALTER TABLE jobs ADD COLUMN disk_written INTEGER;\n\
94         ALTER TABLE jobs ADD COLUMN disk_write_ios INTEGER;\n\
95         ALTER TABLE jobs ADD COLUMN disk_dirty INTEGER;\n\
96         ALTER TABLE jobs ADD COLUMN disk_measure TEXT;\n\
97         ALTER TABLE runners ADD COLUMN cgroup_gone_at INTEGER;\n\
98         ALTER TABLE samples ADD COLUMN written INTEGER;\n\
99         ALTER TABLE samples ADD COLUMN write_ios INTEGER;\n\
100         ALTER TABLE samples ADD COLUMN dirty INTEGER;\n\
101         CREATE TABLE disk_windows (start INTEGER PRIMARY KEY NOT NULL, end_at INTEGER NOT NULL, \
102         device TEXT NOT NULL, store TEXT, drive INTEGER, slice INTEGER, daemon INTEGER, commits INTEGER) WITHOUT ROWID;\n\
103         CREATE INDEX jobs_by_runner ON jobs (runner);",
104    ),
105    (25, "ALTER TABLE jobs ADD COLUMN failure_cause TEXT;"),
106    (
107        26,
108        "ALTER TABLE jobs ADD COLUMN wait_causes TEXT;\n\
109         ALTER TABLE jobs ADD COLUMN unseen_unlisted INTEGER NOT NULL DEFAULT 0;",
110    ),
111    (
112        27,
113        "ALTER TABLE jobs ADD COLUMN rerun_at INTEGER;\n\
114         CREATE TABLE rerun_claims (repository TEXT NOT NULL, run_id INTEGER NOT NULL, attempt INTEGER NOT NULL, \
115         claimed_at INTEGER NOT NULL, PRIMARY KEY (repository, run_id, attempt)) WITHOUT ROWID;",
116    ),
117    (28, "ALTER TABLE runners ADD COLUMN probe TEXT;"),
118];
119
120/// The schema version this build reads and writes, the database's `user_version`: its last migration's.
121const VERSION: u32 = match MIGRATIONS.last() {
122    Some((version, _)) => *version,
123    None => OLDEST,
124};
125
126/// Bytes of log lines sent and not yet stored, at most: a job's capture waits beyond it, so a noisy job or a slow
127/// disk holds back that job's log, not the daemon's memory.
128pub(crate) const LOG_BACKLOG: u64 = 8 << 20;
129
130/// Host samples the writer holds before it stores them in one transaction: a minute's worth. One transaction per
131/// sample wrote about 300 MB a day to an idle daemon's WAL. A crash loses the samples held.
132const HOST_SAMPLE_BATCH: usize = 30;
133
134/// How many events a slow client may fall behind before its stream ends; it reconnects with `Last-Event-ID`.
135const EVENT_BUFFER: usize = 4096;
136
137/// Why the database cannot be used.
138#[derive(Debug, thiserror::Error)]
139pub enum StoreError {
140    /// SQLite failed.
141    #[error("{0}")]
142    Sqlite(#[from] rusqlite::Error),
143    /// A newer runnerd wrote the database.
144    #[error(
145        "the job history database is version {found}, newer than the {supported} this runnerd reads; \
146         run a newer runnerd or move the database aside"
147    )]
148    Newer {
149        /// The database's version.
150        found: u32,
151        /// The newest version this build knows.
152        supported: u32,
153    },
154    /// A runnerd before 0.0.19 wrote the database.
155    #[error(
156        "the job history database is version {found}, from a runnerd before 0.0.19; this runnerd reads versions \
157         from {oldest} on: move the database aside"
158    )]
159    Older {
160        /// The database's version.
161        found: u32,
162        /// The oldest version this build reads.
163        oldest: u32,
164    },
165    /// The directory or file cannot be created.
166    #[error("{0}")]
167    Io(#[from] std::io::Error),
168    /// A blocking read did not finish.
169    #[error("a database read stopped: {0}")]
170    Join(#[from] tokio::task::JoinError),
171    /// The database's `-wal` file holds changes not yet in it, and this user cannot open the database the way that
172    /// reads them.
173    #[error(
174        "{} holds changes not yet in the job history, which this user cannot read; run runnerd doctor as the user \
175         the daemon runs as",
176        .0.display()
177    )]
178    Unread(PathBuf),
179}
180
181impl StoreError {
182    /// Whether a read stopped because the runtime shuts down, as when the daemon exits, rather than because the
183    /// history failed: the runtime cancels the blocking reads it has not run.
184    pub fn is_shutdown(&self) -> bool {
185        matches!(self, StoreError::Join(join) if join.is_cancelled())
186    }
187}
188
189/// A stretch of one job's wait for one cause, which the history adds to the job's `wait_causes` (ADR 0154).
190#[derive(Debug, Clone, Copy, PartialEq, Eq)]
191pub struct CauseWait {
192    /// The job.
193    pub job: u64,
194    /// What held it.
195    pub cause: api::WaitCause,
196    /// For how long, in ms.
197    pub ms: u64,
198}
199
200/// A runner container as the history records it.
201#[derive(Debug, Clone, PartialEq, Eq)]
202pub struct RunnerRecord {
203    /// The runner's name, also its container's.
204    pub name: String,
205    /// `owner/name`.
206    pub repository: String,
207    /// The profile it runs in.
208    pub profile: String,
209    /// 1-based.
210    pub slot: usize,
211    /// The Docker container ID.
212    pub container: String,
213    /// When the container started, in milliseconds since the epoch.
214    pub started_at: u64,
215    /// The container's memory cap in bytes.
216    pub memory_cap: Option<u64>,
217    /// The container's CPU limit in millicores.
218    pub cpu_cap: Option<u64>,
219    /// How it was admitted; `None` for one a restart adopted without a record of it.
220    pub admission: Option<api::Admission>,
221    /// Why it started: `job`, `warm` or `burst`; `None` for one a restart adopted.
222    pub purpose: Option<&'static str>,
223}
224
225/// One sample of one runner's container.
226#[derive(Debug, Clone, PartialEq, Eq)]
227pub struct SampleRecord {
228    /// The runner.
229    pub runner: String,
230    /// When, in milliseconds since the epoch.
231    pub at: u64,
232    /// Millicores over the interval before it.
233    pub cpu: u64,
234    /// CPU time so far, in microseconds.
235    pub cpu_usage: u64,
236    /// Bytes without page cache (`observe::metrics::Usage::memory`).
237    pub memory: u64,
238    /// Bytes received since the container started.
239    pub network_received: u64,
240    /// Bytes sent since the container started.
241    pub network_sent: u64,
242    /// The container's `memory.events` so far.
243    pub events: Events,
244    /// What held the container's CPU back so far.
245    pub cpu_held: CpuHeld,
246    /// What it wrote to the disks under Docker's root so far; `None` where it was not read.
247    pub written: Option<Written>,
248    /// Page cache it had written and the kernel not yet to the disk; `None` where it was not read.
249    pub dirty: Option<u64>,
250}
251
252/// The last measurement of the cache volumes.
253#[derive(Debug, Clone, Default, PartialEq, Eq)]
254pub struct Measured {
255    /// When it was taken (seconds); none before the first measurement.
256    pub at: Option<u64>,
257    /// Each volume's size in bytes, by name.
258    pub bytes: BTreeMap<String, u64>,
259}
260
261/// The profiles and repositories as the stored events last announced them: what clients resuming after those
262/// events have.
263#[derive(Debug, Clone, Default, PartialEq, Eq)]
264pub struct Announced {
265    /// The last `profile.changed` of each profile not removed since, by name.
266    pub profiles: Vec<api::ProfileStatus>,
267    /// The last `repository.changed` of each repository not removed since, by name.
268    pub repositories: Vec<api::RepositoryStatus>,
269}
270
271/// A job the history has not closed, as the engine needs it after a restart.
272#[derive(Debug, Clone, PartialEq, Eq)]
273pub struct OpenJob {
274    /// GitHub's job ID.
275    pub id: u64,
276    /// `owner/name`.
277    pub repository: String,
278    /// `Queued` or `Running`.
279    pub state: api::JobState,
280    /// Its runner, once one took it.
281    pub runner: Option<String>,
282    /// When its runner was removed, in milliseconds since the epoch.
283    pub runner_removed_at: Option<u64>,
284    /// The workflow run it belongs to.
285    pub run_id: u64,
286    /// 1 for the run's first attempt.
287    pub run_attempt: u32,
288    /// The workflow's name, when known.
289    pub workflow: Option<String>,
290    /// When GitHub queued it, in milliseconds since the epoch.
291    pub queued_at: u64,
292    /// The labels it asked for, which tell its repository's pool.
293    pub labels: Vec<String>,
294}
295
296/// The job a slot ran last, as the history has it.
297#[derive(Debug, Clone, PartialEq, Eq)]
298pub struct SlotJob {
299    /// The profile the slot is in.
300    pub profile: String,
301    /// 1-based.
302    pub slot: usize,
303    /// The job, as admission tells jobs apart.
304    pub key: crate::admission::Key,
305}
306
307/// Where a job's log capture continues: the runner's page file and the byte offset in it.
308#[derive(Debug, Clone, Copy, PartialEq, Eq)]
309pub struct LogPosition {
310    /// The page number, from 1.
311    pub page: u64,
312    /// Bytes of that page already stored as lines.
313    pub offset: u64,
314}
315
316/// How far the mirror's log was counted: the time of the last response counted, in nanoseconds since the epoch as
317/// Docker logs it, and how many responses logged at exactly that time were counted, so the next read counts the
318/// ones it was too early to see.
319#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
320pub struct MirrorPosition {
321    /// Nanoseconds since the epoch.
322    pub at: u64,
323    /// Responses logged at `at` already counted.
324    pub seen: u64,
325}
326
327/// What the writer thread is asked to do, in order.
328#[derive(Debug)]
329enum Write {
330    Queued {
331        repository: String,
332        job: QueuedJob,
333        seen_at: u64,
334    },
335    RunnerStarted(RunnerRecord),
336    RunnerAdmission {
337        runner: String,
338        admission: api::Admission,
339    },
340    RunnerOomKilled(String),
341    RunnerProbe {
342        runner: String,
343        probe: api::RunnerProbe,
344    },
345    RunnerStoppedFor {
346        runner: String,
347        reason: &'static str,
348    },
349    RunnerCancelBy {
350        runner: String,
351        at: u64,
352    },
353    HostWaited {
354        jobs: Vec<u64>,
355        ms: u64,
356    },
357    CausesWaited(Vec<CauseWait>),
358    SlotWaited {
359        jobs: Vec<u64>,
360        ms: u64,
361    },
362    Reran {
363        jobs: Vec<u64>,
364        at: u64,
365    },
366    BuildCache {
367        runner: String,
368        counts: api::BuildCacheCounts,
369    },
370    RunnerRemoved {
371        runner: String,
372        at: u64,
373    },
374    RunnersGone {
375        running: Vec<String>,
376        at: u64,
377    },
378    Claimed {
379        job: u64,
380        run_id: u64,
381        runner: String,
382        at: u64,
383    },
384    Details {
385        details: JobDetails,
386    },
387    WorkflowNamed(NamedRun),
388    Elsewhere {
389        details: JobDetails,
390    },
391    Lost {
392        job: u64,
393        at: u64,
394    },
395    Unanswered {
396        job: u64,
397        at: u64,
398    },
399    Log {
400        job: u64,
401        lines: Vec<String>,
402        /// Where the capture resumes once these lines are stored; `None` for all but the last part of a batch.
403        position: Option<LogPosition>,
404        max_bytes: u64,
405        /// The lines' share of `LOG_BACKLOG`, given back once they are stored.
406        _backlog: OwnedSemaphorePermit,
407    },
408    Samples(Vec<SampleRecord>),
409    CgroupsGone {
410        runners: Vec<String>,
411        at: u64,
412    },
413    DiskWindow(Window),
414    HostSample(api::HostSample),
415    MirrorBytes {
416        bytes: Vec<(String, u64)>,
417        until: MirrorPosition,
418        /// Whether the count committed.
419        done: oneshot::Sender<bool>,
420    },
421    CacheSizes {
422        at: u64,
423        sizes: BTreeMap<String, u64>,
424    },
425    /// Boxed: an event with a job's summary is many times the other writes' size.
426    Event(Box<Event>),
427    RecordMade(api::Record),
428    RecordSkipped {
429        job: u64,
430        why: String,
431    },
432    RecordRemoved {
433        job: u64,
434        at: u64,
435        by: api::RecordRemoval,
436    },
437    RecordsListed {
438        jobs: Vec<u64>,
439        at: u64,
440    },
441    Paused {
442        repositories: Vec<String>,
443        /// Whether it committed.
444        done: oneshot::Sender<bool>,
445    },
446    Visibility {
447        repository: String,
448        visibility: api::Visibility,
449        at: u64,
450    },
451    SizingIntent {
452        change: api::SizingChange,
453        /// Its row, once it committed.
454        done: oneshot::Sender<Option<i64>>,
455    },
456    SizingResolved {
457        id: i64,
458        resolution: Resolution,
459        /// Whether it committed.
460        done: oneshot::Sender<bool>,
461    },
462    Notify {
463        change: NotifyChange,
464        /// Whether it committed.
465        done: oneshot::Sender<bool>,
466    },
467    RunnerNamed {
468        secs: u64,
469        /// Whether it committed.
470        done: oneshot::Sender<bool>,
471    },
472    ClaimRerun {
473        attempt: run::ClaimKey,
474        at: u64,
475        done: oneshot::Sender<Claim>,
476    },
477    ReleaseRerun(run::ClaimKey),
478    Flush(oneshot::Sender<()>),
479}
480
481/// What the notifier records.
482#[derive(Debug)]
483enum NotifyChange {
484    /// Sets a target's keys and removes others.
485    Record {
486        target: String,
487        set: Vec<(String, u64)>,
488        remove: Vec<String>,
489    },
490    /// Forgets every target but these.
491    Keep(Vec<String>),
492}
493
494/// What became of a pending auto sizing change.
495#[derive(Debug, Clone, Copy, PartialEq, Eq)]
496pub enum Resolution {
497    /// The config file has it: announced as `sizing.changed`.
498    Done,
499    /// The file was not written.
500    Failed,
501    /// Left pending, and the config no longer says whether it was written: it keeps its interval and stays listed,
502    /// with what it replaced.
503    Uncertain,
504}
505
506/// Auto sizing as the history keeps it: when each profile last changed, and changes not yet known to be written.
507#[derive(Debug, Clone, Default, PartialEq, Eq)]
508pub struct AutoSizing {
509    /// The latest pending or done change of each profile, in seconds, however old.
510    pub last: BTreeMap<String, u64>,
511    /// Changes recorded before their file was written and not resolved since, by row.
512    pub pending: Vec<(i64, api::SizingChange)>,
513    /// Each profile's `cpus` as the daemon ran them, from its `profile.changed` events, oldest first: those `[history]
514    /// events` keeps, and each profile's latest however old.
515    pub cpus: Vec<sizing::CpusSet>,
516    /// Each profile's `memory` as the daemon ran it, from the same events.
517    pub memory: Vec<sizing::MemorySet>,
518    /// Each profile's slots as the daemon ran them, from the same events: one more for each slot a lowering left a
519    /// runner in until it ends.
520    pub slots: Vec<sizing::SlotsSet>,
521}
522
523/// The job history. Cloning shares it.
524#[derive(Debug, Clone)]
525pub struct Store {
526    writes: mpsc::UnboundedSender<Write>,
527    reader: Arc<Mutex<Connection>>,
528    /// Taken with `try_lock`, so one prune runs at a time; one that panicked midway left rows the next one deletes.
529    pruner: Arc<Mutex<Connection>>,
530    /// Set by `close`: a running prune stops after its batch, and none starts.
531    closed: Arc<AtomicBool>,
532    /// Host samples sent and not yet stored, oldest first; reads include them. The writer removes them once stored.
533    // Locked with `sync::lock`: every critical section is a push, a copy or a drain.
534    held: Arc<Mutex<Vec<api::HostSample>>>,
535    /// The last host sample sent, for admission to read without a query.
536    latest_host: Arc<Mutex<Option<api::HostSample>>>,
537    /// The last reading of the host's memory, which is not stored.
538    latest_host_memory: Arc<Mutex<Option<api::HostMemory>>>,
539    /// The jobs the last plan had waiting for the host's memory, and when (ms); insights read it.
540    host_waiting: Arc<Mutex<(BTreeSet<u64>, u64)>>,
541    /// Each running runner's last sample, for admission to read without a query.
542    latest_samples: Arc<Mutex<BTreeMap<String, SampleRecord>>>,
543    /// The runners whose container's cgroup the sampler saw go, until they are removed or it reads them again.
544    gone: Arc<Mutex<BTreeSet<String>>>,
545    events: broadcast::Sender<Arc<EventEnvelope>>,
546    backlog: Arc<Semaphore>,
547}
548
549impl Store {
550    /// Opens the database at `path`, creating it and its directory (mode 0700) if missing, with the schema when it
551    /// is new, and brings it to the schema this build reads. Refuses a database from before 0.0.19 or from a newer
552    /// runnerd. The time since the history's last sign of life is taken as time no runnerd ran: the jobs queued in
553    /// it leave it out of their waits.
554    pub fn open(path: &Path) -> Result<Store, StoreError> {
555        Store::open_at(path, crate::time::now_ms())
556    }
557
558    /// [`Store::open`] at `now` (ms).
559    fn open_at(path: &Path, now: u64) -> Result<Store, StoreError> {
560        if let Some(dir) = path.parent().filter(|dir| !dir.as_os_str().is_empty()) {
561            std::fs::DirBuilder::new()
562                .recursive(true)
563                .mode(0o700)
564                .create(dir)?;
565        }
566        // Owner-only, as SQLite then makes its `-wal` and `-shm` files: logs may say more than the owner wants shared.
567        std::fs::OpenOptions::new()
568            .create(true)
569            .append(true)
570            .mode(0o600)
571            .open(path)?;
572        let mut db = Connection::open(path)?;
573        configure(&db)?;
574        prepare(&mut db)?;
575        let downtime = Downtime::before(&mut db, now)?;
576        db.execute("INSERT INTO sessions (started_at) VALUES (?1)", [now])?;
577        let session = db.last_insert_rowid();
578        let reader = Connection::open_with_flags(
579            path,
580            OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX,
581        )?;
582        configure(&reader)?;
583        let pruner = Connection::open(path)?;
584        configure(&pruner)?;
585        let (events, _) = broadcast::channel(EVENT_BUFFER);
586        let (writes, receiver) = mpsc::unbounded_channel();
587        let thread_events = events.clone();
588        let held = Arc::new(Mutex::new(Vec::new()));
589        let thread_held = held.clone();
590        std::thread::Builder::new()
591            .name("runnerd-store".to_owned())
592            .spawn(move || {
593                write::run(
594                    write::Started {
595                        db,
596                        downtime,
597                        session,
598                    },
599                    receiver,
600                    &thread_events,
601                    &thread_held,
602                );
603            })?;
604        Ok(Store {
605            writes,
606            reader: Arc::new(Mutex::new(reader)),
607            pruner: Arc::new(Mutex::new(pruner)),
608            closed: Arc::new(AtomicBool::new(false)),
609            held,
610            latest_host: Arc::new(Mutex::new(None)),
611            latest_host_memory: Arc::new(Mutex::new(None)),
612            host_waiting: Arc::new(Mutex::new((BTreeSet::new(), 0))),
613            latest_samples: Arc::new(Mutex::new(BTreeMap::new())),
614            gone: Arc::new(Mutex::new(BTreeSet::new())),
615            events,
616            backlog: Arc::new(Semaphore::new(
617                usize::try_from(LOG_BACKLOG).unwrap_or(usize::MAX),
618            )),
619        })
620    }
621
622    fn send(&self, write: Write) {
623        // Fails only once the writer thread is gone, after a SQLite error it logged.
624        let _ = self.writes.send(write);
625    }
626
627    /// Records a queued job the repository's runners may run, which a listing answered at `seen_at` (ms) showed;
628    /// announces it the first time, and keeps the first listing's time. A job a runner claimed first keeps its state
629    /// and gets `seen_at` if no listing showed it before and this one answered before it started.
630    pub fn queued(&self, repository: &str, job: &QueuedJob, seen_at: u64) {
631        self.send(Write::Queued {
632            repository: repository.to_owned(),
633            job: job.clone(),
634            seen_at,
635        });
636    }
637
638    /// Records a runner container that started, or one an earlier daemon started and this one adopted.
639    pub fn runner_started(&self, runner: RunnerRecord) {
640        self.send(Write::RunnerStarted(runner));
641    }
642
643    /// Records that a runner's container and registration were removed at `at` (ms).
644    pub fn runner_removed(&self, runner: &str, at: u64) {
645        lock(&self.latest_samples).remove(runner);
646        lock(&self.gone).remove(runner);
647        self.send(Write::RunnerRemoved {
648            runner: runner.to_owned(),
649            at,
650        });
651    }
652
653    /// Records every runner the history holds open, except the `running` ones, removed at `at` (ms): what recovery
654    /// found gone, so it stops counting as holding a slot.
655    pub fn runners_gone(&self, running: Vec<String>, at: u64) {
656        lock(&self.latest_samples).retain(|runner, _| running.contains(runner));
657        lock(&self.gone).retain(|runner| running.contains(runner));
658        self.send(Write::RunnersGone { running, at });
659    }
660
661    /// Records that a runner took a job at `at` (ms).
662    pub fn claimed(&self, job: u64, run_id: u64, runner: &str, at: u64) {
663        self.send(Write::Claimed {
664            job,
665            run_id,
666            runner: runner.to_owned(),
667            at,
668        });
669    }
670
671    /// Records a running job's progress and steps as GitHub reports them; announces changed steps.
672    pub fn details(&self, details: JobDetails) {
673        self.send(Write::Details { details });
674    }
675
676    /// Names the workflow of a run's jobs, which were recorded under GitHub's name for the run because the workflow's
677    /// could not be read then (ADR 0140).
678    pub fn workflow_named(&self, named: NamedRun) {
679        self.send(Write::WorkflowNamed(named));
680    }
681
682    /// Closes a queued job that left the queue without a runner of this daemon: another machine's runner took it, or
683    /// none did and it has no start.
684    pub fn elsewhere(&self, details: JobDetails) {
685        self.send(Write::Elsewhere { details });
686    }
687
688    /// Closes a queued job that left the queue and that GitHub no longer answers for, at `at`: none of this daemon's
689    /// runners took it, so it reads as run elsewhere, with no conclusion.
690    pub fn unanswered(&self, job: u64, at: u64) {
691        self.send(Write::Unanswered { job, at });
692    }
693
694    /// Closes a job whose container is gone and that GitHub did not report completed in time.
695    pub fn lost(&self, job: u64, at: u64) {
696        self.send(Write::Lost { job, at });
697    }
698
699    /// Appends log lines to a job, up to `max_bytes` per job, and records where the capture continues. Waits while
700    /// `LOG_BACKLOG` bytes of lines are waiting to be stored; a larger batch goes in parts, each waiting its turn.
701    pub async fn log(&self, job: u64, lines: Vec<String>, position: LogPosition, max_bytes: u64) {
702        let size = |line: &String| line.len() as u64 + 1;
703        let mut lines = lines.into_iter().map(|mut line| {
704            // No line takes the whole backlog; the capture already cuts lines far shorter.
705            if size(&line) > LOG_BACKLOG {
706                let mut end = usize::try_from(LOG_BACKLOG / 2).unwrap_or(usize::MAX);
707                while !line.is_char_boundary(end) {
708                    end -= 1;
709                }
710                line.truncate(end);
711            }
712            line
713        });
714        let mut next = lines.next();
715        loop {
716            let (mut part, mut bytes) = (Vec::new(), 0);
717            while let Some(line) = next.take_if(|line| bytes + size(line) <= LOG_BACKLOG) {
718                bytes += size(&line);
719                part.push(line);
720                next = lines.next();
721            }
722            let last = next.is_none();
723            let share = u32::try_from(bytes.max(1)).unwrap_or(u32::MAX);
724            // The semaphore is never closed.
725            let Ok(backlog) = self.backlog.clone().acquire_many_owned(share).await else {
726                return;
727            };
728            self.send(Write::Log {
729                job,
730                lines: part,
731                position: last.then_some(position),
732                max_bytes,
733                _backlog: backlog,
734            });
735            if last {
736                return;
737            }
738        }
739    }
740
741    /// Records samples of runner containers.
742    pub fn samples(&self, samples: Vec<SampleRecord>) {
743        let mut latest = lock(&self.latest_samples);
744        for sample in &samples {
745            latest.insert(sample.runner.clone(), sample.clone());
746        }
747        drop(latest);
748        self.send(Write::Samples(samples));
749    }
750
751    /// Records a sample of the slice and sends it to every subscriber at once; it is stored with the next ones, a
752    /// minute's worth in one transaction, or at `flush`.
753    pub fn host_sample(&self, sample: api::HostSample) {
754        *lock(&self.latest_host) = Some(sample.clone());
755        lock(&self.held).push(sample.clone());
756        self.send(Write::HostSample(sample));
757    }
758
759    /// A runner's last sample, stored or not; `None` before its first sample or once it is removed.
760    pub fn latest_sample(&self, runner: &str) -> Option<SampleRecord> {
761        lock(&self.latest_samples).get(runner).cloned()
762    }
763
764    /// Notes that the cgroups of these runners' containers are gone (`metrics::Sampled::gone`), for admission to read
765    /// until they are removed. Kept in memory only: a restart finds such a container stopped.
766    pub fn containers_gone(&self, runners: Vec<String>) {
767        lock(&self.gone).extend(runners);
768    }
769
770    /// Records that these runners' containers' cgroups were gone at `at` (ms), one sample after they were last read
771    /// (`metrics::Sampled::ended`): what they wrote is measured to their end (ADR 0143). A job already closed gets
772    /// its figure again.
773    pub fn containers_ended(&self, runners: Vec<String>, at: u64) {
774        if !runners.is_empty() {
775            self.send(Write::CgroupsGone { runners, at });
776        }
777    }
778
779    /// Records a window of what the disks under Docker's root wrote.
780    pub fn disk_window(&self, window: Window) {
781        self.send(Write::DiskWindow(window));
782    }
783
784    /// Notes that these runners' containers' cgroups were read (`metrics::Sampled::read`): one marked gone came back,
785    /// as a `docker restart` brings it, and holds memory again.
786    pub fn containers_read(&self, runners: &[String]) {
787        let mut gone = lock(&self.gone);
788        for runner in runners {
789            gone.remove(runner);
790        }
791    }
792
793    /// Whether a runner's container's cgroup is gone, though its slot may hold it a while yet.
794    pub fn container_gone(&self, runner: &str) -> bool {
795        lock(&self.gone).contains(runner)
796    }
797
798    /// Records what a runner reserves now, when measured admission changed it after the start.
799    pub fn runner_admission(&self, runner: &str, admission: api::Admission) {
800        self.send(Write::RunnerAdmission {
801            runner: runner.to_owned(),
802            admission,
803        });
804    }
805
806    /// Records that Docker saw the kernel kill a process in a runner's container for memory, which its samples may
807    /// have missed: one in the last interval before the container stopped.
808    pub fn runner_oom_killed(&self, runner: &str) {
809        self.send(Write::RunnerOomKilled(runner.to_owned()));
810    }
811
812    /// Records what the probe after a runner exited found (ADR 0155), as its stopped container says.
813    pub fn runner_probe(&self, runner: &str, probe: api::RunnerProbe) {
814        self.send(Write::RunnerProbe {
815            runner: runner.to_owned(),
816            probe,
817        });
818    }
819
820    /// Records when a forced pause that cancelled a runner's run removes it if it still runs, in ms since the epoch,
821    /// so a restart keeps that deadline.
822    pub fn runner_cancel_by(&self, runner: &str, at: u64) {
823        self.send(Write::RunnerCancelBy {
824            runner: runner.to_owned(),
825            at,
826        });
827    }
828
829    /// Records why an idle runner was removed, as `scheduler::StopReason::as_str` names it.
830    pub fn runner_stopped_for(&self, runner: &str, reason: &'static str) {
831        self.send(Write::RunnerStoppedFor {
832            runner: runner.to_owned(),
833            reason,
834        });
835    }
836
837    /// Adds what the shared build cache served a runner and took from it, and records whether its job could write.
838    pub fn build_cache(&self, runner: &str, counts: api::BuildCacheCounts) {
839        self.send(Write::BuildCache {
840            runner: runner.to_owned(),
841            counts,
842        });
843    }
844
845    /// Keeps the latest reading of the host's memory for admission and the status; it is not stored.
846    pub fn host_memory(&self, memory: api::HostMemory) {
847        *lock(&self.latest_host_memory) = Some(memory);
848    }
849
850    /// The latest reading of the host's memory; `None` before the first.
851    pub fn latest_host_memory(&self) -> Option<api::HostMemory> {
852        lock(&self.latest_host_memory).clone()
853    }
854
855    /// Keeps which `jobs` the plan at `at` (ms) had waiting for memory the host lacks (ADR 0103), for insights to read
856    /// with the reading; it is not stored. A plan that finds none replaces the last one's.
857    pub fn host_waiting(&self, jobs: BTreeSet<u64>, at: u64) {
858        *lock(&self.host_waiting) = (jobs, at);
859    }
860
861    /// Adds `ms` to how long each of `jobs` waited because the host's memory could not take its runner, which
862    /// sizing does not read as a wait for a slot.
863    pub fn host_waited(&self, jobs: Vec<u64>, ms: u64) {
864        self.send(Write::HostWaited { jobs, ms });
865    }
866
867    /// Adds `ms` to how long each of `jobs` waited with no slot of its profile free while the budgets had room for
868    /// its runner: what auto sizing reads as a wait for a slot (ADR 0090).
869    pub fn slot_waited(&self, jobs: Vec<u64>, ms: u64) {
870        self.send(Write::SlotWaited { jobs, ms });
871    }
872
873    /// Adds what each wait in `waits` lasted to its job's time for its cause (ADR 0154), all in one transaction and
874    /// with one write of each job's row. A job whose row is from before schema 26 keeps counting none, so it never
875    /// reads as a wait of 0.
876    pub fn causes_waited(&self, waits: Vec<CauseWait>) {
877        if waits.iter().any(|wait| wait.ms > 0) {
878            self.send(Write::CausesWaited(waits));
879        }
880    }
881
882    /// Notes that runnerd asked GitHub to re-run the failed jobs of the run `jobs` are of because their runner failed
883    /// them, and GitHub accepted, at `at` ms (ADR 0153). A job noted already keeps its first time.
884    pub fn reran(&self, jobs: Vec<u64>, at: u64) {
885        self.send(Write::Reran { jobs, at });
886    }
887
888    /// Claims the automatic re-run of `repository`'s run `run` from attempt `attempt`, at `at` ms, before runnerd sends
889    /// it (ADR 0153), synced to disk: a restart reads no fault of a claimed attempt back, and a second claim of it is
890    /// [`Claim::Taken`]. A claim outlives the history's mark, which only a re-run GitHub accepted writes. Claims a day
891    /// older than `at` go with it: no fault that old is read back.
892    pub async fn claim_rerun(&self, repository: &str, run: u64, attempt: u32, at: u64) -> Claim {
893        let (done, claimed) = oneshot::channel();
894        self.send(Write::ClaimRerun {
895            attempt: run::ClaimKey {
896                repository: repository.to_owned(),
897                run,
898                attempt,
899            },
900            at,
901            done,
902        });
903        // Waited for on a blocking thread, as `runner_named` is.
904        tokio::task::spawn_blocking(move || claimed.blocking_recv().unwrap_or(Claim::Unstored))
905            .await
906            .unwrap_or(Claim::Unstored)
907    }
908
909    /// Releases the claim [`Store::claim_rerun`] made, once nothing of the re-run reached GitHub: the permit stopped
910    /// it, or GitHub refused it. A release lost to a crash leaves the attempt claimed, never sent.
911    pub fn release_rerun(&self, repository: &str, run: u64, attempt: u32) {
912        self.send(Write::ReleaseRerun(run::ClaimKey {
913            repository: repository.to_owned(),
914            run,
915            attempt,
916        }));
917    }
918
919    /// The last host sample sent, stored or not; `None` before the first.
920    pub fn latest_host_sample(&self) -> Option<api::HostSample> {
921        lock(&self.latest_host).clone()
922    }
923
924    /// Adds bytes the mirror served to runners, counted from its log up to `until`, and says whether that was
925    /// stored: if not, the caller counts the same lines again.
926    pub async fn mirror_bytes(&self, bytes: Vec<(String, u64)>, until: MirrorPosition) -> bool {
927        let (done, committed) = oneshot::channel();
928        self.send(Write::MirrorBytes { bytes, until, done });
929        committed.await.unwrap_or(false)
930    }
931
932    /// Records volume sizes in bytes, by name, measured at `at` (ms).
933    pub fn cache_sizes(&self, at: u64, sizes: BTreeMap<String, u64>) {
934        self.send(Write::CacheSizes { at, sizes });
935    }
936
937    /// Stores a durable event, or sends a live one, to every subscriber.
938    pub fn event(&self, event: Event) {
939        self.send(Write::Event(Box::new(event)));
940    }
941
942    /// Records that a failed job's runner was kept as `record`, and announces it.
943    pub fn record_made(&self, record: api::Record) {
944        self.send(Write::RecordMade(record));
945    }
946
947    /// Records why a failed job's runner was not kept.
948    pub fn record_skipped(&self, job: u64, why: String) {
949        self.send(Write::RecordSkipped { job, why });
950    }
951
952    /// Records that a job's record was removed at `at` (ms), and why, and announces it.
953    pub fn record_removed(&self, job: u64, at: u64, by: api::RecordRemoval) {
954        self.send(Write::RecordRemoved { job, at, by });
955    }
956
957    /// Records that Docker listed the records of `jobs`, and no others, at `at` (ms): a record the history notes as
958    /// kept whose job is not among them was removed outside runnerd, and is noted gone and announced as
959    /// `record.gone`. Only an inventory Docker answered is sent; one it did not changes nothing.
960    pub fn records_listed(&self, jobs: Vec<u64>, at: u64) {
961        self.send(Write::RecordsListed { jobs, at });
962    }
963
964    /// Records the paused repositories, all of them, in place of the ones recorded before. The write is sent at
965    /// once, in order with every other; the future says whether it committed.
966    pub fn paused(&self, repositories: Vec<String>) -> impl Future<Output = bool> + use<> {
967        let (done, committed) = oneshot::channel();
968        self.send(Write::Paused { repositories, done });
969        async move { committed.await.unwrap_or(false) }
970    }
971
972    /// Records that this machine named a runner for `secs`, the second its name ends in, before GitHub or Docker
973    /// knows the name: a restart names the next runner later even when nothing else recorded this one. Says whether
974    /// it committed.
975    pub async fn runner_named(&self, secs: u64) -> bool {
976        let (done, committed) = oneshot::channel();
977        self.send(Write::RunnerNamed { secs, done });
978        // Waited for on a blocking thread, as a read is: the runtime then knows the start waits for the disk, and a
979        // paused test clock does not run on meanwhile.
980        tokio::task::spawn_blocking(move || committed.blocking_recv().unwrap_or(false))
981            .await
982            .unwrap_or(false)
983    }
984
985    /// Records a repository's visibility as GitHub gave it at `at` (ms), in place of the one recorded before.
986    pub fn visibility(&self, repository: String, visibility: api::Visibility, at: u64) {
987        self.send(Write::Visibility {
988            repository,
989            visibility,
990            at,
991        });
992    }
993
994    /// Records a change auto sizing is about to write, as pending, before the file is, synced to disk. The future
995    /// has its row once it committed, `None` if it did not: then nothing may be written.
996    pub fn sizing_intent(
997        &self,
998        change: api::SizingChange,
999    ) -> impl Future<Output = Option<i64>> + use<> {
1000        let (done, committed) = oneshot::channel();
1001        self.send(Write::SizingIntent { change, done });
1002        async move { committed.await.ok().flatten() }
1003    }
1004
1005    /// Records what became of a pending change, synced to disk; a done one is announced as `sizing.changed` in the
1006    /// same transaction. The future says whether it committed.
1007    pub fn sizing_resolved(
1008        &self,
1009        id: i64,
1010        resolution: Resolution,
1011    ) -> impl Future<Output = bool> + use<> {
1012        let (done, committed) = oneshot::channel();
1013        self.send(Write::SizingResolved {
1014            id,
1015            resolution,
1016            done,
1017        });
1018        async move { committed.await.unwrap_or(false) }
1019    }
1020
1021    /// Records what the notifier delivered to a target: sets `set`'s keys and removes `remove`'s, in one
1022    /// transaction. The future says whether it committed.
1023    pub fn notify_record(
1024        &self,
1025        target: &str,
1026        set: Vec<(String, u64)>,
1027        remove: Vec<String>,
1028    ) -> impl Future<Output = bool> + use<> {
1029        let (done, committed) = oneshot::channel();
1030        self.send(Write::Notify {
1031            change: NotifyChange::Record {
1032                target: target.to_owned(),
1033                set,
1034                remove,
1035            },
1036            done,
1037        });
1038        async move { committed.await.unwrap_or(false) }
1039    }
1040
1041    /// Forgets what the notifier kept for every target but `targets`. The future says whether it committed.
1042    pub fn notify_keep(&self, targets: Vec<String>) -> impl Future<Output = bool> + use<> {
1043        let (done, committed) = oneshot::channel();
1044        self.send(Write::Notify {
1045            change: NotifyChange::Keep(targets),
1046            done,
1047        });
1048        async move { committed.await.unwrap_or(false) }
1049    }
1050
1051    /// Deletes what is older than `history` keeps, at `now` (ms), off the writer, in batches the writer's
1052    /// transactions fit between. A prune while the last one runs does nothing. The handle finishes with it; dropping
1053    /// it lets it run.
1054    pub fn prune(&self, now: u64, history: &History) -> tokio::task::JoinHandle<()> {
1055        let pruner = self.pruner.clone();
1056        let closed = self.closed.clone();
1057        let history = history.clone();
1058        tokio::task::spawn_blocking(move || {
1059            let db = match pruner.try_lock() {
1060                Ok(db) => db,
1061                Err(TryLockError::Poisoned(poisoned)) => poisoned.into_inner(),
1062                Err(TryLockError::WouldBlock) => return,
1063            };
1064            let db = prune::Db {
1065                connection: &db,
1066                stop: &closed,
1067            };
1068            if !db.stop.load(Ordering::Relaxed) {
1069                prune::run(&db, now, &history);
1070            }
1071        })
1072    }
1073
1074    /// Stops a running prune after its batch, and any later one before it starts, so a shutdown does not wait for
1075    /// one. What a stopped prune left, the next daemon's deletes. Writes go on as before.
1076    pub fn close(&self) {
1077        self.closed.store(true, Ordering::Relaxed);
1078    }
1079
1080    /// Waits until every write sent before is done.
1081    pub async fn flush(&self) {
1082        let (done, wait) = oneshot::channel();
1083        self.send(Write::Flush(done));
1084        let _ = wait.await;
1085    }
1086
1087    /// Every event from now on, durable or live. Subscribe before reading what was stored, then skip what the read
1088    /// returned, and nothing is missed or sent twice.
1089    pub fn subscribe(&self) -> broadcast::Receiver<Arc<EventEnvelope>> {
1090        self.events.subscribe()
1091    }
1092
1093    async fn read<T: Send + 'static>(
1094        &self,
1095        query: impl FnOnce(&Connection) -> rusqlite::Result<T> + Send + 'static,
1096    ) -> Result<T, StoreError> {
1097        let reader = self.reader.clone();
1098        // A panic during a read leaves the connection usable: reads change nothing.
1099        Ok(tokio::task::spawn_blocking(move || query(&lock(&reader))).await??)
1100    }
1101
1102    /// One page of jobs as `query` asks, which [`api::JobQuery::check`] passed (ADR 0135).
1103    pub async fn jobs(&self, query: api::JobQuery) -> Result<api::JobList, StoreError> {
1104        self.read(move |db| read::jobs(db, &query)).await
1105    }
1106
1107    /// What each slot ran over `window` (ADR 0135), with the host samples not yet stored counted as watched time.
1108    pub async fn timeline(&self, window: api::Span) -> Result<api::SlotTimeline, StoreError> {
1109        let held: Vec<u64> = lock(&self.held).iter().map(|sample| sample.at).collect();
1110        self.read(move |db| lens::timeline(db, window, &held, api::SAMPLE_INTERVAL))
1111            .await
1112    }
1113
1114    /// Job `id` beside the job `with` names, or its baseline (ADR 0135); `None` without either job.
1115    pub async fn compare(
1116        &self,
1117        id: u64,
1118        with: Option<u64>,
1119    ) -> Result<Option<api::JobComparison>, StoreError> {
1120        self.read(move |db| lens::compare(db, id, with)).await
1121    }
1122
1123    /// A job with its steps.
1124    pub async fn job(&self, id: u64) -> Result<Option<api::Job>, StoreError> {
1125        self.read(move |db| read::job(db, id)).await
1126    }
1127
1128    /// The jobs their runner failed (`runner_fault`) whose records became final at or after `since` ms and whose
1129    /// attempt runnerd has neither re-run nor claimed, oldest first: what a restarted daemon may still re-run (ADR 0153).
1130    pub async fn runner_faults(&self, since: u64) -> Result<Vec<RunnerFault>, StoreError> {
1131        self.read(move |db| run::runner_faults(db, since)).await
1132    }
1133
1134    /// Up to `limit` other jobs of `repository`'s run `run` than `except`, newest attempt first (ADR 0134).
1135    pub async fn run_jobs(
1136        &self,
1137        repository: &str,
1138        run: u64,
1139        except: u64,
1140        limit: usize,
1141    ) -> Result<RunJobs, StoreError> {
1142        let repository = repository.to_owned();
1143        self.read(move |db| run::run_jobs(db, &repository, run, except, limit))
1144            .await
1145    }
1146
1147    /// A job's samples, oldest first; `None` if there is no such job.
1148    pub async fn job_samples(
1149        &self,
1150        id: u64,
1151        interval: Duration,
1152    ) -> Result<Option<api::SampleSeries>, StoreError> {
1153        self.read(move |db| read::job_samples(db, id, interval))
1154            .await
1155    }
1156
1157    /// Up to `limit` log lines of a job from line `from`, with their numbers.
1158    pub async fn log_lines(
1159        &self,
1160        job: u64,
1161        from: u64,
1162        limit: u64,
1163    ) -> Result<Vec<(u64, String)>, StoreError> {
1164        self.read(move |db| read::log_lines(db, job, from, limit))
1165            .await
1166    }
1167
1168    /// A job's log as structure (ADR 0133); `None` without the job.
1169    pub async fn log_outline(&self, job: u64) -> Result<Option<api::LogOutline>, StoreError> {
1170        self.read(move |db| outline::outline(db, job)).await
1171    }
1172
1173    /// Up to `limit` styled lines of a job's log from line `from`, fewer once they hold 4 MiB of text; each shows at most
1174    /// [`api::LINE_BYTES_MAX`] bytes of its line.
1175    pub async fn styled_lines(
1176        &self,
1177        job: u64,
1178        from: u64,
1179        limit: u64,
1180    ) -> Result<Vec<api::StyledLine>, StoreError> {
1181        self.read(move |db| outline::styled_lines(db, job, from, limit))
1182            .await
1183    }
1184
1185    /// One page of a search of a job's log, each batch of lines its own read, so the search never holds the reader
1186    /// for a whole log.
1187    pub async fn search_log(
1188        &self,
1189        job: u64,
1190        mut search: api::Search,
1191    ) -> Result<api::LogSearch, StoreError> {
1192        // Each batch is styled and marked where it is read, off the async workers.
1193        while let Some(want) = search.wants() {
1194            search = self
1195                .read(move |db| {
1196                    search.read(read::lines_from(db, job, want)?);
1197                    Ok(search)
1198                })
1199                .await?;
1200        }
1201        Ok(search.finish())
1202    }
1203
1204    /// Up to `limit` durable events after `after`, oldest first.
1205    pub async fn events_after(
1206        &self,
1207        after: u64,
1208        limit: u64,
1209    ) -> Result<Vec<EventEnvelope>, StoreError> {
1210        self.read(move |db| read::events_after(db, after, limit))
1211            .await
1212    }
1213
1214    /// The changes auto sizing wrote since `since` (ms), oldest first.
1215    pub async fn sizing_changes(&self, since: u64) -> Result<Vec<api::SizingChange>, StoreError> {
1216        self.read(move |db| read::sizing_changes(db, since)).await
1217    }
1218
1219    /// When each profile last changed, and the changes still pending.
1220    pub async fn auto_sizing(&self) -> Result<AutoSizing, StoreError> {
1221        self.read(read::auto_sizing).await
1222    }
1223
1224    /// The profiles and repositories as the stored events last announced them, so a restarted daemon announces
1225    /// only what changed since, including what its config no longer has. Blocks on the database: for startup.
1226    pub fn announced(&self) -> Result<Announced, StoreError> {
1227        Ok(read::announced(&lock(&self.reader))?)
1228    }
1229
1230    /// The repositories recorded paused, for the engine to start with. Blocks on the database: for startup.
1231    pub fn paused_repositories(&self) -> Result<Vec<String>, StoreError> {
1232        Ok(read::paused(&lock(&self.reader))?)
1233    }
1234
1235    /// The slice's samples since `since` (seconds), oldest first, the ones not yet stored included.
1236    pub async fn host_samples(&self, since: u64) -> Result<Vec<api::HostSample>, StoreError> {
1237        // Copied before the read: the writer removes a sample only once it is stored, so each one is in the copy,
1238        // the read, or both.
1239        let held = lock(&self.held).clone();
1240        let mut samples = self.read(move |db| read::host_samples(db, since)).await?;
1241        let stored = samples.last().map_or(0, |sample| sample.at);
1242        samples.extend(
1243            held.into_iter()
1244                .filter(|sample| sample.at >= since && sample.at > stored),
1245        );
1246        Ok(samples)
1247    }
1248
1249    /// The jobs runnerd's runners started and finished from `since` until `now` (seconds).
1250    pub async fn activity(&self, since: u64, now: u64) -> Result<api::Activity, StoreError> {
1251        self.read(move |db| read::activity(db, since, now)).await
1252    }
1253
1254    /// Finished jobs per repository and bucket.
1255    pub async fn stats(&self, query: api::StatsQuery, now: u64) -> Result<api::Stats, StoreError> {
1256        self.read(move |db| read::stats(db, &query, now)).await
1257    }
1258
1259    /// What sizing reads: the finished jobs queued at or after `since` (ms) and the runners since.
1260    pub async fn sizing(&self, since: u64) -> Result<sizing::History, StoreError> {
1261        self.read(move |db| read::sizing(db, since)).await
1262    }
1263
1264    /// What pricing at GitHub's rates reads: the completed and lost jobs dated at or after `since` (ms), each
1265    /// repository's visibility and since when the history holds every final job.
1266    pub async fn pricing(&self, since: u64) -> Result<api::PricingHistory, StoreError> {
1267        self.read(move |db| read::pricing(db, since)).await
1268    }
1269
1270    /// The jobs completed or lost that were queued at or after `since` (ms), as insights read them, from when the
1271    /// history holds every job, and the queued jobs the last plan had waiting for the host's memory.
1272    pub async fn insights(&self, since: u64) -> Result<crate::insights::History, StoreError> {
1273        let memory = self.latest_host_memory();
1274        let (waiting, seen) = lock(&self.host_waiting).clone();
1275        self.read(move |db| {
1276            let mut history = read::insights(db, since)?;
1277            if let Some(memory) = memory.filter(|_| !waiting.is_empty()) {
1278                let jobs = read::held_jobs(db, &waiting)?;
1279                history.host =
1280                    (!jobs.is_empty()).then_some(crate::insights::HostHold { memory, seen, jobs });
1281            }
1282            Ok(history)
1283        })
1284        .await
1285    }
1286
1287    /// The newest durable event's ID; 0 before the first.
1288    pub async fn last_event(&self) -> Result<u64, StoreError> {
1289        self.read(read::last_event).await
1290    }
1291
1292    /// What the notifier kept for each target, by target, then key.
1293    pub async fn notify_state(
1294        &self,
1295    ) -> Result<BTreeMap<String, BTreeMap<String, u64>>, StoreError> {
1296        self.read(read::notify_state).await
1297    }
1298
1299    /// The jobs queued at or after `since` (ms), whatever became of them, as warm runners learn from them.
1300    pub async fn warm(&self, since: u64) -> Result<Vec<warm::Seen>, StoreError> {
1301        self.read(move |db| read::warm(db, since)).await
1302    }
1303
1304    /// The last measurement of every volume, with its time (seconds), by name.
1305    pub async fn cache_sizes_now(&self) -> Result<Measured, StoreError> {
1306        self.read(read::latest_cache_sizes).await
1307    }
1308
1309    /// What the build cache served and took from the runners started since `since` (ms), summed.
1310    pub async fn build_cache_since(&self, since: u64) -> Result<api::BuildCacheCounts, StoreError> {
1311        self.read(move |db| read::build_cache_since(db, since))
1312            .await
1313    }
1314
1315    /// Every volume measurement since `since` (seconds).
1316    pub async fn cache_history(&self, since: u64) -> Result<Vec<api::CacheSize>, StoreError> {
1317        self.read(move |db| read::cache_history(db, since)).await
1318    }
1319
1320    /// The jobs not yet closed, for the engine after a restart.
1321    pub async fn open_jobs(&self) -> Result<Vec<OpenJob>, StoreError> {
1322        self.read(read::open_jobs).await
1323    }
1324
1325    /// The newest named job each slot ran, for the engine after a restart.
1326    pub async fn slot_jobs(&self) -> Result<Vec<SlotJob>, StoreError> {
1327        self.read(read::slot_jobs).await
1328    }
1329
1330    /// The latest second a runner was named for (`runner_named`), or any runner name starting with `prefix` carries,
1331    /// of a runner or of a job's runner: the engine names runners later after a restart, so no name it used before
1332    /// is used twice.
1333    pub async fn last_runner_second(&self, prefix: String) -> Result<Option<u64>, StoreError> {
1334        self.read(move |db| read::last_runner_second(db, &prefix))
1335            .await
1336    }
1337
1338    /// The job a runner took, if the history knows it.
1339    pub async fn runner_job(&self, runner: &str) -> Result<Option<u64>, StoreError> {
1340        let runner = runner.to_owned();
1341        self.read(move |db| read::runner_job(db, &runner)).await
1342    }
1343
1344    /// How a runner was admitted, with what it reserves since, if the history knows it.
1345    pub async fn runner_admitted(
1346        &self,
1347        runner: &str,
1348    ) -> Result<Option<api::Admission>, StoreError> {
1349        let runner = runner.to_owned();
1350        self.read(move |db| read::runner_admission(db, &runner))
1351            .await
1352    }
1353
1354    /// When a forced pause that cancelled a runner's run removes it if it still runs, in ms since the epoch, if one
1355    /// did.
1356    pub async fn runner_cancel_deadline(&self, runner: &str) -> Result<Option<u64>, StoreError> {
1357        let runner = runner.to_owned();
1358        self.read(move |db| read::runner_cancel_deadline(db, &runner))
1359            .await
1360    }
1361
1362    /// The memory cap a runner started with, as its record kept it, for when Docker cannot tell.
1363    pub async fn runner_memory_cap(&self, runner: &str) -> Result<Option<u64>, StoreError> {
1364        let runner = runner.to_owned();
1365        self.read(move |db| read::runner_memory_cap(db, &runner))
1366            .await
1367    }
1368
1369    /// Where a job's log capture continues.
1370    pub async fn log_position(&self, job: u64) -> Result<LogPosition, StoreError> {
1371        self.read(move |db| read::log_position(db, job)).await
1372    }
1373
1374    /// How far the mirror's log was counted.
1375    pub async fn mirror_log_until(&self) -> Result<MirrorPosition, StoreError> {
1376        self.read(read::mirror_log_until).await
1377    }
1378}
1379
1380/// What sizing reads from the database at `path`, opened read-only and left as it is, as `runnerd doctor` reads it
1381/// while no daemon runs; `None` when there is no database yet. In a directory it may write, SQLite keeps the
1382/// `-wal` and `-shm` files a read of a WAL database needs next to it; in one it may not, it reads the file as it is,
1383/// or refuses when the `-wal` holds changes the file does not have yet.
1384pub fn sizing_at(path: &Path, since: u64) -> Result<Option<sizing::History>, StoreError> {
1385    read_only(path, |db| sizing_of(db, since))
1386}
1387
1388/// How the jobs of the history at `path` that started from `since` (ms) ran, read as [`sizing_at`] reads, for a replay
1389/// that runs them so (`admission::simulate`): their samples, when the daemon could name each, what the host's programs
1390/// held. `overhead` is what the slice holds beside its runners, in bytes; `None` measures it from the history's busy
1391/// moments, as `read::trace` says. `named_by` is the longest a known job waits to be read: the poll interval.
1392pub fn trace_at(
1393    path: &Path,
1394    since: u64,
1395    named_by: u64,
1396    overhead: Option<u64>,
1397) -> Result<Option<Trace>, StoreError> {
1398    read_only(path, |db| {
1399        if version(db)? == 0 {
1400            return Ok(Trace::default());
1401        }
1402        Ok(read::trace(db, since, named_by, overhead)?)
1403    })
1404}
1405
1406/// When the last job of the history at `path` was queued, in ms, read as [`sizing_at`] reads; `None` with no job yet.
1407/// A replay's window counts back from it, so a backup replays its own last hours.
1408pub fn last_queued_at(path: &Path) -> Result<Option<u64>, StoreError> {
1409    let last = read_only(path, |db| {
1410        if version(db)? == 0 {
1411            return Ok(None);
1412        }
1413        Ok(db.query_row("SELECT MAX(queued_at) FROM jobs", [], |row| row.get(0))?)
1414    })?;
1415    Ok(last.flatten())
1416}
1417
1418/// What `read` finds in the database at `path`, opened read-only and left as it is; `None` when there is none.
1419fn read_only<T>(
1420    path: &Path,
1421    read: impl Fn(&Connection) -> Result<T, StoreError>,
1422) -> Result<Option<T>, StoreError> {
1423    if !path.exists() {
1424        return Ok(None);
1425    }
1426    let read = |path: &str, flags| -> Result<T, StoreError> {
1427        let db = Connection::open_with_flags(path, flags)?;
1428        db.busy_timeout(Duration::from_secs(5))?;
1429        read(&db)
1430    };
1431    let name = path.to_string_lossy();
1432    match read(&name, OpenFlags::SQLITE_OPEN_READ_ONLY) {
1433        Err(StoreError::Sqlite(error))
1434            if matches!(
1435                error.sqlite_error_code(),
1436                Some(rusqlite::ErrorCode::ReadOnly | rusqlite::ErrorCode::CannotOpen)
1437            ) =>
1438        {
1439            // No `-shm` can be made: read the file alone, unless the `-wal` holds what it lacks.
1440            let mut wal = path.as_os_str().to_owned();
1441            wal.push("-wal");
1442            let wal = PathBuf::from(wal);
1443            if std::fs::metadata(&wal).is_ok_and(|wal| wal.len() > 0) {
1444                return Err(StoreError::Unread(wal));
1445            }
1446            let uri = format!(
1447                "file:{}?mode=ro&immutable=1",
1448                name.replace('%', "%25")
1449                    .replace('?', "%3f")
1450                    .replace('#', "%23")
1451            );
1452            read(
1453                &uri,
1454                OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_URI,
1455            )
1456        }
1457        history => history,
1458    }
1459    .map(Some)
1460}
1461
1462/// What sizing reads from the database, without creating its schema; an older one reads as [`VERSION`].
1463fn sizing_of(db: &Connection, since: u64) -> Result<sizing::History, StoreError> {
1464    match version(db)? {
1465        0 => return Ok(sizing::History::default()),
1466        VERSION => {}
1467        _ => read_as_current(db)?,
1468    }
1469    let mut history = read::sizing(db, since)?;
1470    let auto = read::auto_sizing(db)?;
1471    history.memory_set = auto.memory;
1472    history.slots_set = auto.slots;
1473    history.changes = read::sizing_changes(db, since.saturating_sub(sizing::memory::HISTORY))?;
1474    Ok(history)
1475}
1476
1477/// The database's schema version: 0 for a new one, else from [`OLDEST`] to [`VERSION`]; any other is refused.
1478fn version(db: &Connection) -> Result<u32, StoreError> {
1479    let found: u32 = db.pragma_query_value(None, "user_version", |row| row.get(0))?;
1480    match found {
1481        0 | OLDEST..=VERSION => Ok(found),
1482        found if found > VERSION => Err(StoreError::Newer {
1483            found,
1484            supported: VERSION,
1485        }),
1486        found => Err(StoreError::Older {
1487            found,
1488            oldest: OLDEST,
1489        }),
1490    }
1491}
1492
1493/// Lets a read-only look at a database from before [`VERSION`], which it cannot migrate, read it as this build
1494/// does: a table it lacks becomes an empty temporary one, and a table that lacks columns a temporary view of it
1495/// that adds them, each with its default. Temporary objects come first in name lookups.
1496fn read_as_current(db: &Connection) -> Result<(), StoreError> {
1497    let current = Connection::open_in_memory()?;
1498    current.execute_batch(SCHEMA)?;
1499    let tables: Vec<(String, String)> = current
1500        .prepare(
1501            "SELECT name, sql FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'",
1502        )?
1503        .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?
1504        .collect::<rusqlite::Result<_>>()?;
1505    let columns =
1506        |db: &Connection, table: &str| -> rusqlite::Result<Vec<(String, Option<String>)>> {
1507            db.prepare("SELECT name, dflt_value FROM pragma_table_info(?1, 'main')")?
1508                .query_map([table], |row| Ok((row.get(0)?, row.get(1)?)))?
1509                .collect()
1510        };
1511    for (table, sql) in tables {
1512        let found: Vec<String> = columns(db, &table)?
1513            .into_iter()
1514            .map(|(name, _)| name)
1515            .collect();
1516        if found.is_empty() {
1517            db.execute_batch(&sql.replacen("CREATE TABLE", "CREATE TEMP TABLE", 1))?;
1518            continue;
1519        }
1520        let added: Vec<String> = columns(&current, &table)?
1521            .into_iter()
1522            .filter(|(name, _)| !found.contains(name))
1523            .map(|(name, default)| format!("{} AS {name}", default.as_deref().unwrap_or("NULL")))
1524            .collect();
1525        if !added.is_empty() {
1526            db.execute_batch(&format!(
1527                "CREATE TEMP VIEW {table} AS SELECT *, {} FROM main.{table}",
1528                added.join(", ")
1529            ))?;
1530        }
1531    }
1532    Ok(())
1533}
1534
1535/// WAL, so the API reads while the writer writes; a busy timeout instead of an immediate error.
1536fn configure(db: &Connection) -> rusqlite::Result<()> {
1537    db.pragma_update(None, "journal_mode", "wal")?;
1538    db.pragma_update(None, "synchronous", "normal")?;
1539    db.pragma_update(None, "foreign_keys", "on")?;
1540    db.busy_timeout(Duration::from_secs(5))
1541}
1542
1543/// Creates the schema in a new database, in one transaction with its version, or runs the migrations an older one
1544/// lacks, each in its own transaction with the version it brings it to.
1545fn prepare(db: &mut Connection) -> Result<(), StoreError> {
1546    let found = version(db)?;
1547    if found == 0 {
1548        let transaction = db.transaction()?;
1549        transaction.execute_batch(SCHEMA)?;
1550        transaction.pragma_update(None, "user_version", VERSION)?;
1551        transaction.commit()?;
1552        return Ok(());
1553    }
1554    for (version, migration) in MIGRATIONS.iter().filter(|(version, _)| *version > found) {
1555        let transaction = db.transaction()?;
1556        transaction.execute_batch(migration)?;
1557        transaction.pragma_update(None, "user_version", version)?;
1558        transaction.commit()?;
1559    }
1560    Ok(())
1561}
1562
1563/// The time before this daemon when no runnerd ran, in ms: from the history's last sign of life, its newest host
1564/// sample, event or open, to this daemon's start. Host samples are stored a minute's worth at a time, so a crash
1565/// may count up to a minute more. Without a sign of life, as in a new history, there is none: the history's start
1566/// clips the waits of jobs queued before it (`history_kept`).
1567#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1568struct Downtime {
1569    from: u64,
1570    until: u64,
1571}
1572
1573impl Downtime {
1574    /// The downtime before a daemon that starts at `now`; the jobs still queued in the history leave it out of
1575    /// their waits, as the ones first seen after do when they are recorded. The open is a sign of life itself, kept
1576    /// with them: a start that fails after it, and is retried, does not hand out the same downtime again.
1577    fn before(db: &mut Connection, now: u64) -> Result<Option<Downtime>, StoreError> {
1578        // Immediate: it reads, then writes. A deferred transaction that has read cannot wait for the write lock, and
1579        // fails at once while another connection, such as a dropped store's writer finishing, holds it.
1580        let transaction = db.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?;
1581        let last: Option<u64> = transaction.query_row(
1582            "SELECT max(at) FROM (SELECT max(at) AS at FROM host_samples UNION ALL SELECT max(at) FROM events \
1583             UNION ALL SELECT value FROM meta WHERE key = 'opened_at')",
1584            [],
1585            |row| row.get(0),
1586        )?;
1587        let downtime = last
1588            .filter(|last| *last < now)
1589            .map(|from| Downtime { from, until: now });
1590        if let Some(downtime) = downtime {
1591            let queued: Vec<(u64, u64)> = transaction
1592                .prepare("SELECT id, queued_at FROM jobs WHERE state = 'queued'")?
1593                .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?
1594                .collect::<rusqlite::Result<_>>()?;
1595            for (job, queued_at) in queued {
1596                transaction.execute(
1597                    "UPDATE jobs SET unseen = unseen + ?2 WHERE id = ?1",
1598                    rusqlite::params![job, downtime.of(queued_at)],
1599                )?;
1600            }
1601        }
1602        transaction.execute(
1603            "INSERT OR REPLACE INTO meta (key, value) VALUES ('opened_at', ?1)",
1604            [now],
1605        )?;
1606        transaction.commit()?;
1607        Ok(downtime)
1608    }
1609
1610    /// How much of it a job queued at `queued_at` (ms) waited through.
1611    fn of(self, queued_at: u64) -> u64 {
1612        self.until.saturating_sub(queued_at.max(self.from))
1613    }
1614
1615    /// How much of it passed from `from` to `until` (ms).
1616    fn during(self, from: u64, until: u64) -> u64 {
1617        until.min(self.until).saturating_sub(from.max(self.from))
1618    }
1619
1620    /// How much of what [`Downtime::of`] gave a job queued at `queued_at` came after it started at `started_at`
1621    /// (ms), as for a job a runner that outlived the last daemon took while none ran.
1622    fn after(self, queued_at: u64, started_at: u64) -> u64 {
1623        self.of(queued_at)
1624            .saturating_sub(started_at.saturating_sub(queued_at.max(self.from)))
1625    }
1626}
1627
1628/// Logs a failed write; the writer goes on with the next one.
1629fn logged<T>(what: &str, result: rusqlite::Result<T>) -> Option<T> {
1630    result
1631        .map_err(|error| warn!("job history: cannot {what}: {error}"))
1632        .ok()
1633}
1634
1635#[cfg(test)]
1636mod tests;