Skip to main content

runnerd_core/
store.rs

1//! The job history: a SQLite database of jobs, their runners, steps, samples and logs, the events clients resume
2//! from, cache sizes, the paused repositories and where the notifier got to. Owns the schema, the one writer and
3//! the event broadcast. Writes are messages to a writer thread, so the engine never waits on the disk; reads and
4//! pruning run on Tokio's blocking pool, pruning on its own connection, a batch at a time. It decides nothing about
5//! scheduling and stores no secret: no token, no JIT config, no runner `_diag` log.
6
7use std::collections::{BTreeMap, BTreeSet};
8use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt};
9use std::path::{Path, PathBuf};
10use std::sync::atomic::{AtomicBool, Ordering};
11use std::sync::{Arc, Mutex, TryLockError};
12use std::time::Duration;
13
14use rusqlite::{Connection, OpenFlags};
15use tokio::sync::{OwnedSemaphorePermit, Semaphore, broadcast, mpsc, oneshot};
16use tracing::warn;
17
18use crate::admission::trace::Trace;
19use crate::api::{self, Event, EventEnvelope};
20use crate::config::History;
21use crate::github::{JobDetails, NamedRun, QueuedJob};
22use crate::observe::metrics::{CpuHeld, Events};
23use crate::scheduler::warm;
24use crate::sizing;
25use crate::sync::lock;
26
27mod lens;
28mod outline;
29mod prune;
30mod read;
31mod run;
32mod write;
33
34pub use run::RunJobs;
35
36/// The schema a new database is created with, at [`VERSION`].
37const SCHEMA: &str = include_str!("store/schema.sql");
38
39/// The oldest schema version a database is brought up from: 0.0.19's (ADR 0073).
40const OLDEST: u32 = 15;
41
42/// The steps from [`OLDEST`] on, in order, each with the version it brings a database to. A change to the schema
43/// adds one here and makes the same change to `schema.sql`, which a new database gets instead.
44const MIGRATIONS: &[(u32, &str)] = &[
45    (
46        16,
47        "ALTER TABLE jobs ADD COLUMN unseen INTEGER NOT NULL DEFAULT 0;",
48    ),
49    (
50        17,
51        "ALTER TABLE jobs ADD COLUMN cpu_periods INTEGER;\n\
52         ALTER TABLE jobs ADD COLUMN cpu_throttled INTEGER;\n\
53         ALTER TABLE jobs ADD COLUMN cpu_stalled INTEGER;\n\
54         ALTER TABLE samples ADD COLUMN cpu_periods INTEGER;\n\
55         ALTER TABLE samples ADD COLUMN cpu_throttled INTEGER;\n\
56         ALTER TABLE samples ADD COLUMN cpu_stalled INTEGER;",
57    ),
58    (
59        18,
60        "ALTER TABLE jobs ADD COLUMN slot_waited INTEGER NOT NULL DEFAULT 0;",
61    ),
62    (
63        19,
64        "ALTER TABLE jobs ADD COLUMN seen_at INTEGER;\n\
65         ALTER TABLE runners ADD COLUMN purpose TEXT;\n\
66         ALTER TABLE runners ADD COLUMN stopped_for TEXT;",
67    ),
68    (
69        20,
70        "ALTER TABLE jobs ADD COLUMN runner_fault INTEGER NOT NULL DEFAULT 0;",
71    ),
72    (21, "ALTER TABLE runners ADD COLUMN cancel_by INTEGER;"),
73    (
74        22,
75        "CREATE TABLE log_outlines (job INTEGER PRIMARY KEY NOT NULL, from_line INTEGER NOT NULL, \
76         errors INTEGER NOT NULL, warnings INTEGER NOT NULL, groups_kept INTEGER NOT NULL, \
77         marks_kept INTEGER NOT NULL, sections_kept INTEGER NOT NULL, dropped INTEGER NOT NULL, \
78         hidden INTEGER NOT NULL);\n\
79         CREATE TABLE log_groups (job INTEGER NOT NULL, start INTEGER NOT NULL, end_line INTEGER, \
80         depth INTEGER NOT NULL, title TEXT NOT NULL, PRIMARY KEY (job, start)) WITHOUT ROWID;\n\
81         CREATE INDEX log_groups_open ON log_groups (job, start) WHERE end_line IS NULL;\n\
82         CREATE TABLE log_marks (job INTEGER NOT NULL, line INTEGER NOT NULL, kind TEXT NOT NULL, \
83         PRIMARY KEY (job, line)) WITHOUT ROWID;",
84    ),
85    (
86        23,
87        "CREATE TABLE sessions (id INTEGER PRIMARY KEY, started_at INTEGER NOT NULL, sampled_at INTEGER);\n\
88         ALTER TABLE records ADD COLUMN gone_at INTEGER;",
89    ),
90];
91
92/// The schema version this build reads and writes, the database's `user_version`: its last migration's.
93const VERSION: u32 = match MIGRATIONS.last() {
94    Some((version, _)) => *version,
95    None => OLDEST,
96};
97
98/// Bytes of log lines sent and not yet stored, at most: a job's capture waits beyond it, so a noisy job or a slow
99/// disk holds back that job's log, not the daemon's memory.
100pub(crate) const LOG_BACKLOG: u64 = 8 << 20;
101
102/// Host samples the writer holds before it stores them in one transaction: a minute's worth. One transaction per
103/// sample wrote about 300 MB a day to an idle daemon's WAL. A crash loses the samples held.
104const HOST_SAMPLE_BATCH: usize = 30;
105
106/// How many events a slow client may fall behind before its stream ends; it reconnects with `Last-Event-ID`.
107const EVENT_BUFFER: usize = 4096;
108
109/// Why the database cannot be used.
110#[derive(Debug, thiserror::Error)]
111pub enum StoreError {
112    /// SQLite failed.
113    #[error("{0}")]
114    Sqlite(#[from] rusqlite::Error),
115    /// A newer runnerd wrote the database.
116    #[error(
117        "the job history database is version {found}, newer than the {supported} this runnerd reads; \
118         run a newer runnerd or move the database aside"
119    )]
120    Newer {
121        /// The database's version.
122        found: u32,
123        /// The newest version this build knows.
124        supported: u32,
125    },
126    /// A runnerd before 0.0.19 wrote the database.
127    #[error(
128        "the job history database is version {found}, from a runnerd before 0.0.19; this runnerd reads versions \
129         from {oldest} on: move the database aside"
130    )]
131    Older {
132        /// The database's version.
133        found: u32,
134        /// The oldest version this build reads.
135        oldest: u32,
136    },
137    /// The directory or file cannot be created.
138    #[error("{0}")]
139    Io(#[from] std::io::Error),
140    /// A blocking read did not finish.
141    #[error("a database read stopped: {0}")]
142    Join(#[from] tokio::task::JoinError),
143    /// The database's `-wal` file holds changes not yet in it, and this user cannot open the database the way that
144    /// reads them.
145    #[error(
146        "{} holds changes not yet in the job history, which this user cannot read; run runnerd doctor as the user \
147         the daemon runs as",
148        .0.display()
149    )]
150    Unread(PathBuf),
151}
152
153impl StoreError {
154    /// Whether a read stopped because the runtime shuts down, as when the daemon exits, rather than because the
155    /// history failed: the runtime cancels the blocking reads it has not run.
156    pub fn is_shutdown(&self) -> bool {
157        matches!(self, StoreError::Join(join) if join.is_cancelled())
158    }
159}
160
161/// A runner container as the history records it.
162#[derive(Debug, Clone, PartialEq, Eq)]
163pub struct RunnerRecord {
164    /// The runner's name, also its container's.
165    pub name: String,
166    /// `owner/name`.
167    pub repository: String,
168    /// The profile it runs in.
169    pub profile: String,
170    /// 1-based.
171    pub slot: usize,
172    /// The Docker container ID.
173    pub container: String,
174    /// When the container started, in milliseconds since the epoch.
175    pub started_at: u64,
176    /// The container's memory cap in bytes.
177    pub memory_cap: Option<u64>,
178    /// The container's CPU limit in millicores.
179    pub cpu_cap: Option<u64>,
180    /// How it was admitted; `None` for one a restart adopted without a record of it.
181    pub admission: Option<api::Admission>,
182    /// Why it started: `job`, `warm` or `burst`; `None` for one a restart adopted.
183    pub purpose: Option<&'static str>,
184}
185
186/// One sample of one runner's container.
187#[derive(Debug, Clone, PartialEq, Eq)]
188pub struct SampleRecord {
189    /// The runner.
190    pub runner: String,
191    /// When, in milliseconds since the epoch.
192    pub at: u64,
193    /// Millicores over the interval before it.
194    pub cpu: u64,
195    /// CPU time so far, in microseconds.
196    pub cpu_usage: u64,
197    /// Bytes without page cache (`observe::metrics::Usage::memory`).
198    pub memory: u64,
199    /// Bytes received since the container started.
200    pub network_received: u64,
201    /// Bytes sent since the container started.
202    pub network_sent: u64,
203    /// The container's `memory.events` so far.
204    pub events: Events,
205    /// What held the container's CPU back so far.
206    pub cpu_held: CpuHeld,
207}
208
209/// The last measurement of the cache volumes.
210#[derive(Debug, Clone, Default, PartialEq, Eq)]
211pub struct Measured {
212    /// When it was taken (seconds); none before the first measurement.
213    pub at: Option<u64>,
214    /// Each volume's size in bytes, by name.
215    pub bytes: BTreeMap<String, u64>,
216}
217
218/// The profiles and repositories as the stored events last announced them: what clients resuming after those
219/// events have.
220#[derive(Debug, Clone, Default, PartialEq, Eq)]
221pub struct Announced {
222    /// The last `profile.changed` of each profile not removed since, by name.
223    pub profiles: Vec<api::ProfileStatus>,
224    /// The last `repository.changed` of each repository not removed since, by name.
225    pub repositories: Vec<api::RepositoryStatus>,
226}
227
228/// A job the history has not closed, as the engine needs it after a restart.
229#[derive(Debug, Clone, PartialEq, Eq)]
230pub struct OpenJob {
231    /// GitHub's job ID.
232    pub id: u64,
233    /// `owner/name`.
234    pub repository: String,
235    /// `Queued` or `Running`.
236    pub state: api::JobState,
237    /// Its runner, once one took it.
238    pub runner: Option<String>,
239    /// When its runner was removed, in milliseconds since the epoch.
240    pub runner_removed_at: Option<u64>,
241    /// The workflow run it belongs to.
242    pub run_id: u64,
243    /// 1 for the run's first attempt.
244    pub run_attempt: u32,
245    /// The workflow's name, when known.
246    pub workflow: Option<String>,
247    /// When GitHub queued it, in milliseconds since the epoch.
248    pub queued_at: u64,
249    /// The labels it asked for, which tell its repository's pool.
250    pub labels: Vec<String>,
251}
252
253/// The job a slot ran last, as the history has it.
254#[derive(Debug, Clone, PartialEq, Eq)]
255pub struct SlotJob {
256    /// The profile the slot is in.
257    pub profile: String,
258    /// 1-based.
259    pub slot: usize,
260    /// The job, as admission tells jobs apart.
261    pub key: crate::admission::Key,
262}
263
264/// Where a job's log capture continues: the runner's page file and the byte offset in it.
265#[derive(Debug, Clone, Copy, PartialEq, Eq)]
266pub struct LogPosition {
267    /// The page number, from 1.
268    pub page: u64,
269    /// Bytes of that page already stored as lines.
270    pub offset: u64,
271}
272
273/// How far the mirror's log was counted: the time of the last response counted, in nanoseconds since the epoch as
274/// Docker logs it, and how many responses logged at exactly that time were counted, so the next read counts the
275/// ones it was too early to see.
276#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
277pub struct MirrorPosition {
278    /// Nanoseconds since the epoch.
279    pub at: u64,
280    /// Responses logged at `at` already counted.
281    pub seen: u64,
282}
283
284/// What the writer thread is asked to do, in order.
285#[derive(Debug)]
286enum Write {
287    Queued {
288        repository: String,
289        job: QueuedJob,
290        seen_at: u64,
291    },
292    RunnerStarted(RunnerRecord),
293    RunnerAdmission {
294        runner: String,
295        admission: api::Admission,
296    },
297    RunnerOomKilled(String),
298    RunnerStoppedFor {
299        runner: String,
300        reason: &'static str,
301    },
302    RunnerCancelBy {
303        runner: String,
304        at: u64,
305    },
306    HostWaited {
307        jobs: Vec<u64>,
308        ms: u64,
309    },
310    SlotWaited {
311        jobs: Vec<u64>,
312        ms: u64,
313    },
314    BuildCache {
315        runner: String,
316        counts: api::BuildCacheCounts,
317    },
318    RunnerRemoved {
319        runner: String,
320        at: u64,
321    },
322    RunnersGone {
323        running: Vec<String>,
324        at: u64,
325    },
326    Claimed {
327        job: u64,
328        run_id: u64,
329        runner: String,
330        at: u64,
331    },
332    Details {
333        details: JobDetails,
334    },
335    WorkflowNamed(NamedRun),
336    Elsewhere {
337        details: JobDetails,
338    },
339    Lost {
340        job: u64,
341        at: u64,
342    },
343    Unanswered {
344        job: u64,
345        at: u64,
346    },
347    Log {
348        job: u64,
349        lines: Vec<String>,
350        /// Where the capture resumes once these lines are stored; `None` for all but the last part of a batch.
351        position: Option<LogPosition>,
352        max_bytes: u64,
353        /// The lines' share of `LOG_BACKLOG`, given back once they are stored.
354        _backlog: OwnedSemaphorePermit,
355    },
356    Samples(Vec<SampleRecord>),
357    HostSample(api::HostSample),
358    MirrorBytes {
359        bytes: Vec<(String, u64)>,
360        until: MirrorPosition,
361        /// Whether the count committed.
362        done: oneshot::Sender<bool>,
363    },
364    CacheSizes {
365        at: u64,
366        sizes: BTreeMap<String, u64>,
367    },
368    /// Boxed: an event with a job's summary is many times the other writes' size.
369    Event(Box<Event>),
370    RecordMade(api::Record),
371    RecordSkipped {
372        job: u64,
373        why: String,
374    },
375    RecordRemoved {
376        job: u64,
377        at: u64,
378        by: api::RecordRemoval,
379    },
380    RecordsListed {
381        jobs: Vec<u64>,
382        at: u64,
383    },
384    Paused {
385        repositories: Vec<String>,
386        /// Whether it committed.
387        done: oneshot::Sender<bool>,
388    },
389    Visibility {
390        repository: String,
391        visibility: api::Visibility,
392        at: u64,
393    },
394    SizingIntent {
395        change: api::SizingChange,
396        /// Its row, once it committed.
397        done: oneshot::Sender<Option<i64>>,
398    },
399    SizingResolved {
400        id: i64,
401        resolution: Resolution,
402        /// Whether it committed.
403        done: oneshot::Sender<bool>,
404    },
405    Notify {
406        change: NotifyChange,
407        /// Whether it committed.
408        done: oneshot::Sender<bool>,
409    },
410    RunnerNamed {
411        secs: u64,
412        /// Whether it committed.
413        done: oneshot::Sender<bool>,
414    },
415    Flush(oneshot::Sender<()>),
416}
417
418/// What the notifier records.
419#[derive(Debug)]
420enum NotifyChange {
421    /// Sets a target's keys and removes others.
422    Record {
423        target: String,
424        set: Vec<(String, u64)>,
425        remove: Vec<String>,
426    },
427    /// Forgets every target but these.
428    Keep(Vec<String>),
429}
430
431/// What became of a pending auto sizing change.
432#[derive(Debug, Clone, Copy, PartialEq, Eq)]
433pub enum Resolution {
434    /// The config file has it: announced as `sizing.changed`.
435    Done,
436    /// The file was not written.
437    Failed,
438    /// Left pending, and the config no longer says whether it was written: it keeps its interval and stays listed,
439    /// with what it replaced.
440    Uncertain,
441}
442
443/// Auto sizing as the history keeps it: when each profile last changed, and changes not yet known to be written.
444#[derive(Debug, Clone, Default, PartialEq, Eq)]
445pub struct AutoSizing {
446    /// The latest pending or done change of each profile, in seconds, however old.
447    pub last: BTreeMap<String, u64>,
448    /// Changes recorded before their file was written and not resolved since, by row.
449    pub pending: Vec<(i64, api::SizingChange)>,
450    /// Each profile's `cpus` as the daemon ran them, from its `profile.changed` events, oldest first: those `[history]
451    /// events` keeps, and each profile's latest however old.
452    pub cpus: Vec<sizing::CpusSet>,
453    /// Each profile's `memory` as the daemon ran it, from the same events.
454    pub memory: Vec<sizing::MemorySet>,
455    /// Each profile's slots as the daemon ran them, from the same events: one more for each slot a lowering left a
456    /// runner in until it ends.
457    pub slots: Vec<sizing::SlotsSet>,
458}
459
460/// The job history. Cloning shares it.
461#[derive(Debug, Clone)]
462pub struct Store {
463    writes: mpsc::UnboundedSender<Write>,
464    reader: Arc<Mutex<Connection>>,
465    /// Taken with `try_lock`, so one prune runs at a time; one that panicked midway left rows the next one deletes.
466    pruner: Arc<Mutex<Connection>>,
467    /// Set by `close`: a running prune stops after its batch, and none starts.
468    closed: Arc<AtomicBool>,
469    /// Host samples sent and not yet stored, oldest first; reads include them. The writer removes them once stored.
470    // Locked with `sync::lock`: every critical section is a push, a copy or a drain.
471    held: Arc<Mutex<Vec<api::HostSample>>>,
472    /// The last host sample sent, for admission to read without a query.
473    latest_host: Arc<Mutex<Option<api::HostSample>>>,
474    /// The last reading of the host's memory, which is not stored.
475    latest_host_memory: Arc<Mutex<Option<api::HostMemory>>>,
476    /// The jobs the last plan had waiting for the host's memory, and when (ms); insights read it.
477    host_waiting: Arc<Mutex<(BTreeSet<u64>, u64)>>,
478    /// Each running runner's last sample, for admission to read without a query.
479    latest_samples: Arc<Mutex<BTreeMap<String, SampleRecord>>>,
480    /// The runners whose container's cgroup the sampler saw go, until they are removed or it reads them again.
481    gone: Arc<Mutex<BTreeSet<String>>>,
482    events: broadcast::Sender<Arc<EventEnvelope>>,
483    backlog: Arc<Semaphore>,
484}
485
486impl Store {
487    /// Opens the database at `path`, creating it and its directory (mode 0700) if missing, with the schema when it
488    /// is new, and brings it to the schema this build reads. Refuses a database from before 0.0.19 or from a newer
489    /// runnerd. The time since the history's last sign of life is taken as time no runnerd ran: the jobs queued in
490    /// it leave it out of their waits.
491    pub fn open(path: &Path) -> Result<Store, StoreError> {
492        Store::open_at(path, crate::time::now_ms())
493    }
494
495    /// [`Store::open`] at `now` (ms).
496    fn open_at(path: &Path, now: u64) -> Result<Store, StoreError> {
497        if let Some(dir) = path.parent().filter(|dir| !dir.as_os_str().is_empty()) {
498            std::fs::DirBuilder::new()
499                .recursive(true)
500                .mode(0o700)
501                .create(dir)?;
502        }
503        // Owner-only, as SQLite then makes its `-wal` and `-shm` files: logs may say more than the owner wants shared.
504        std::fs::OpenOptions::new()
505            .create(true)
506            .append(true)
507            .mode(0o600)
508            .open(path)?;
509        let mut db = Connection::open(path)?;
510        configure(&db)?;
511        prepare(&mut db)?;
512        let downtime = Downtime::before(&mut db, now)?;
513        db.execute("INSERT INTO sessions (started_at) VALUES (?1)", [now])?;
514        let session = db.last_insert_rowid();
515        let reader = Connection::open_with_flags(
516            path,
517            OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX,
518        )?;
519        configure(&reader)?;
520        let pruner = Connection::open(path)?;
521        configure(&pruner)?;
522        let (events, _) = broadcast::channel(EVENT_BUFFER);
523        let (writes, receiver) = mpsc::unbounded_channel();
524        let thread_events = events.clone();
525        let held = Arc::new(Mutex::new(Vec::new()));
526        let thread_held = held.clone();
527        std::thread::Builder::new()
528            .name("runnerd-store".to_owned())
529            .spawn(move || {
530                write::run(
531                    write::Started {
532                        db,
533                        downtime,
534                        session,
535                    },
536                    receiver,
537                    &thread_events,
538                    &thread_held,
539                );
540            })?;
541        Ok(Store {
542            writes,
543            reader: Arc::new(Mutex::new(reader)),
544            pruner: Arc::new(Mutex::new(pruner)),
545            closed: Arc::new(AtomicBool::new(false)),
546            held,
547            latest_host: Arc::new(Mutex::new(None)),
548            latest_host_memory: Arc::new(Mutex::new(None)),
549            host_waiting: Arc::new(Mutex::new((BTreeSet::new(), 0))),
550            latest_samples: Arc::new(Mutex::new(BTreeMap::new())),
551            gone: Arc::new(Mutex::new(BTreeSet::new())),
552            events,
553            backlog: Arc::new(Semaphore::new(
554                usize::try_from(LOG_BACKLOG).unwrap_or(usize::MAX),
555            )),
556        })
557    }
558
559    fn send(&self, write: Write) {
560        // Fails only once the writer thread is gone, after a SQLite error it logged.
561        let _ = self.writes.send(write);
562    }
563
564    /// Records a queued job the repository's runners may run, which a listing answered at `seen_at` (ms) showed;
565    /// announces it the first time, and keeps the first listing's time. A job a runner claimed first keeps its state
566    /// and gets `seen_at` if no listing showed it before and this one answered before it started.
567    pub fn queued(&self, repository: &str, job: &QueuedJob, seen_at: u64) {
568        self.send(Write::Queued {
569            repository: repository.to_owned(),
570            job: job.clone(),
571            seen_at,
572        });
573    }
574
575    /// Records a runner container that started, or one an earlier daemon started and this one adopted.
576    pub fn runner_started(&self, runner: RunnerRecord) {
577        self.send(Write::RunnerStarted(runner));
578    }
579
580    /// Records that a runner's container and registration were removed at `at` (ms).
581    pub fn runner_removed(&self, runner: &str, at: u64) {
582        lock(&self.latest_samples).remove(runner);
583        lock(&self.gone).remove(runner);
584        self.send(Write::RunnerRemoved {
585            runner: runner.to_owned(),
586            at,
587        });
588    }
589
590    /// Records every runner the history holds open, except the `running` ones, removed at `at` (ms): what recovery
591    /// found gone, so it stops counting as holding a slot.
592    pub fn runners_gone(&self, running: Vec<String>, at: u64) {
593        lock(&self.latest_samples).retain(|runner, _| running.contains(runner));
594        lock(&self.gone).retain(|runner| running.contains(runner));
595        self.send(Write::RunnersGone { running, at });
596    }
597
598    /// Records that a runner took a job at `at` (ms).
599    pub fn claimed(&self, job: u64, run_id: u64, runner: &str, at: u64) {
600        self.send(Write::Claimed {
601            job,
602            run_id,
603            runner: runner.to_owned(),
604            at,
605        });
606    }
607
608    /// Records a running job's progress and steps as GitHub reports them; announces changed steps.
609    pub fn details(&self, details: JobDetails) {
610        self.send(Write::Details { details });
611    }
612
613    /// Names the workflow of a run's jobs, which were recorded under GitHub's name for the run because the workflow's
614    /// could not be read then (ADR 0140).
615    pub fn workflow_named(&self, named: NamedRun) {
616        self.send(Write::WorkflowNamed(named));
617    }
618
619    /// Closes a queued job that left the queue without a runner of this daemon: another machine's runner took it, or
620    /// none did and it has no start.
621    pub fn elsewhere(&self, details: JobDetails) {
622        self.send(Write::Elsewhere { details });
623    }
624
625    /// Closes a queued job that left the queue and that GitHub no longer answers for, at `at`: none of this daemon's
626    /// runners took it, so it reads as run elsewhere, with no conclusion.
627    pub fn unanswered(&self, job: u64, at: u64) {
628        self.send(Write::Unanswered { job, at });
629    }
630
631    /// Closes a job whose container is gone and that GitHub did not report completed in time.
632    pub fn lost(&self, job: u64, at: u64) {
633        self.send(Write::Lost { job, at });
634    }
635
636    /// Appends log lines to a job, up to `max_bytes` per job, and records where the capture continues. Waits while
637    /// `LOG_BACKLOG` bytes of lines are waiting to be stored; a larger batch goes in parts, each waiting its turn.
638    pub async fn log(&self, job: u64, lines: Vec<String>, position: LogPosition, max_bytes: u64) {
639        let size = |line: &String| line.len() as u64 + 1;
640        let mut lines = lines.into_iter().map(|mut line| {
641            // No line takes the whole backlog; the capture already cuts lines far shorter.
642            if size(&line) > LOG_BACKLOG {
643                let mut end = usize::try_from(LOG_BACKLOG / 2).unwrap_or(usize::MAX);
644                while !line.is_char_boundary(end) {
645                    end -= 1;
646                }
647                line.truncate(end);
648            }
649            line
650        });
651        let mut next = lines.next();
652        loop {
653            let (mut part, mut bytes) = (Vec::new(), 0);
654            while let Some(line) = next.take_if(|line| bytes + size(line) <= LOG_BACKLOG) {
655                bytes += size(&line);
656                part.push(line);
657                next = lines.next();
658            }
659            let last = next.is_none();
660            let share = u32::try_from(bytes.max(1)).unwrap_or(u32::MAX);
661            // The semaphore is never closed.
662            let Ok(backlog) = self.backlog.clone().acquire_many_owned(share).await else {
663                return;
664            };
665            self.send(Write::Log {
666                job,
667                lines: part,
668                position: last.then_some(position),
669                max_bytes,
670                _backlog: backlog,
671            });
672            if last {
673                return;
674            }
675        }
676    }
677
678    /// Records samples of runner containers.
679    pub fn samples(&self, samples: Vec<SampleRecord>) {
680        let mut latest = lock(&self.latest_samples);
681        for sample in &samples {
682            latest.insert(sample.runner.clone(), sample.clone());
683        }
684        drop(latest);
685        self.send(Write::Samples(samples));
686    }
687
688    /// Records a sample of the slice and sends it to every subscriber at once; it is stored with the next ones, a
689    /// minute's worth in one transaction, or at `flush`.
690    pub fn host_sample(&self, sample: api::HostSample) {
691        *lock(&self.latest_host) = Some(sample.clone());
692        lock(&self.held).push(sample.clone());
693        self.send(Write::HostSample(sample));
694    }
695
696    /// A runner's last sample, stored or not; `None` before its first sample or once it is removed.
697    pub fn latest_sample(&self, runner: &str) -> Option<SampleRecord> {
698        lock(&self.latest_samples).get(runner).cloned()
699    }
700
701    /// Notes that the cgroups of these runners' containers are gone (`metrics::Sampled::gone`), for admission to read
702    /// until they are removed. Kept in memory only: a restart finds such a container stopped.
703    pub fn containers_gone(&self, runners: Vec<String>) {
704        lock(&self.gone).extend(runners);
705    }
706
707    /// Notes that these runners' containers' cgroups were read (`metrics::Sampled::read`): one marked gone came back,
708    /// as a `docker restart` brings it, and holds memory again.
709    pub fn containers_read(&self, runners: &[String]) {
710        let mut gone = lock(&self.gone);
711        for runner in runners {
712            gone.remove(runner);
713        }
714    }
715
716    /// Whether a runner's container's cgroup is gone, though its slot may hold it a while yet.
717    pub fn container_gone(&self, runner: &str) -> bool {
718        lock(&self.gone).contains(runner)
719    }
720
721    /// Records what a runner reserves now, when measured admission changed it after the start.
722    pub fn runner_admission(&self, runner: &str, admission: api::Admission) {
723        self.send(Write::RunnerAdmission {
724            runner: runner.to_owned(),
725            admission,
726        });
727    }
728
729    /// Records that Docker saw the kernel kill a process in a runner's container for memory, which its samples may
730    /// have missed: one in the last interval before the container stopped.
731    pub fn runner_oom_killed(&self, runner: &str) {
732        self.send(Write::RunnerOomKilled(runner.to_owned()));
733    }
734
735    /// Records when a forced pause that cancelled a runner's run removes it if it still runs, in ms since the epoch,
736    /// so a restart keeps that deadline.
737    pub fn runner_cancel_by(&self, runner: &str, at: u64) {
738        self.send(Write::RunnerCancelBy {
739            runner: runner.to_owned(),
740            at,
741        });
742    }
743
744    /// Records why an idle runner was removed, as `scheduler::StopReason::as_str` names it.
745    pub fn runner_stopped_for(&self, runner: &str, reason: &'static str) {
746        self.send(Write::RunnerStoppedFor {
747            runner: runner.to_owned(),
748            reason,
749        });
750    }
751
752    /// Adds what the shared build cache served a runner and took from it, and records whether its job could write.
753    pub fn build_cache(&self, runner: &str, counts: api::BuildCacheCounts) {
754        self.send(Write::BuildCache {
755            runner: runner.to_owned(),
756            counts,
757        });
758    }
759
760    /// Keeps the latest reading of the host's memory for admission and the status; it is not stored.
761    pub fn host_memory(&self, memory: api::HostMemory) {
762        *lock(&self.latest_host_memory) = Some(memory);
763    }
764
765    /// The latest reading of the host's memory; `None` before the first.
766    pub fn latest_host_memory(&self) -> Option<api::HostMemory> {
767        lock(&self.latest_host_memory).clone()
768    }
769
770    /// Keeps which `jobs` the plan at `at` (ms) had waiting for memory the host lacks (ADR 0103), for insights to read
771    /// with the reading; it is not stored. A plan that finds none replaces the last one's.
772    pub fn host_waiting(&self, jobs: BTreeSet<u64>, at: u64) {
773        *lock(&self.host_waiting) = (jobs, at);
774    }
775
776    /// Adds `ms` to how long each of `jobs` waited because the host's memory could not take its runner, which
777    /// sizing does not read as a wait for a slot.
778    pub fn host_waited(&self, jobs: Vec<u64>, ms: u64) {
779        self.send(Write::HostWaited { jobs, ms });
780    }
781
782    /// Adds `ms` to how long each of `jobs` waited with no slot of its profile free while the budgets had room for
783    /// its runner: what auto sizing reads as a wait for a slot (ADR 0090).
784    pub fn slot_waited(&self, jobs: Vec<u64>, ms: u64) {
785        self.send(Write::SlotWaited { jobs, ms });
786    }
787
788    /// The last host sample sent, stored or not; `None` before the first.
789    pub fn latest_host_sample(&self) -> Option<api::HostSample> {
790        lock(&self.latest_host).clone()
791    }
792
793    /// Adds bytes the mirror served to runners, counted from its log up to `until`, and says whether that was
794    /// stored: if not, the caller counts the same lines again.
795    pub async fn mirror_bytes(&self, bytes: Vec<(String, u64)>, until: MirrorPosition) -> bool {
796        let (done, committed) = oneshot::channel();
797        self.send(Write::MirrorBytes { bytes, until, done });
798        committed.await.unwrap_or(false)
799    }
800
801    /// Records volume sizes in bytes, by name, measured at `at` (ms).
802    pub fn cache_sizes(&self, at: u64, sizes: BTreeMap<String, u64>) {
803        self.send(Write::CacheSizes { at, sizes });
804    }
805
806    /// Stores a durable event, or sends a live one, to every subscriber.
807    pub fn event(&self, event: Event) {
808        self.send(Write::Event(Box::new(event)));
809    }
810
811    /// Records that a failed job's runner was kept as `record`, and announces it.
812    pub fn record_made(&self, record: api::Record) {
813        self.send(Write::RecordMade(record));
814    }
815
816    /// Records why a failed job's runner was not kept.
817    pub fn record_skipped(&self, job: u64, why: String) {
818        self.send(Write::RecordSkipped { job, why });
819    }
820
821    /// Records that a job's record was removed at `at` (ms), and why, and announces it.
822    pub fn record_removed(&self, job: u64, at: u64, by: api::RecordRemoval) {
823        self.send(Write::RecordRemoved { job, at, by });
824    }
825
826    /// Records that Docker listed the records of `jobs`, and no others, at `at` (ms): a record the history notes as
827    /// kept whose job is not among them was removed outside runnerd, and is noted gone and announced as
828    /// `record.gone`. Only an inventory Docker answered is sent; one it did not changes nothing.
829    pub fn records_listed(&self, jobs: Vec<u64>, at: u64) {
830        self.send(Write::RecordsListed { jobs, at });
831    }
832
833    /// Records the paused repositories, all of them, in place of the ones recorded before. The write is sent at
834    /// once, in order with every other; the future says whether it committed.
835    pub fn paused(&self, repositories: Vec<String>) -> impl Future<Output = bool> + use<> {
836        let (done, committed) = oneshot::channel();
837        self.send(Write::Paused { repositories, done });
838        async move { committed.await.unwrap_or(false) }
839    }
840
841    /// Records that this machine named a runner for `secs`, the second its name ends in, before GitHub or Docker
842    /// knows the name: a restart names the next runner later even when nothing else recorded this one. Says whether
843    /// it committed.
844    pub async fn runner_named(&self, secs: u64) -> bool {
845        let (done, committed) = oneshot::channel();
846        self.send(Write::RunnerNamed { secs, done });
847        // Waited for on a blocking thread, as a read is: the runtime then knows the start waits for the disk, and a
848        // paused test clock does not run on meanwhile.
849        tokio::task::spawn_blocking(move || committed.blocking_recv().unwrap_or(false))
850            .await
851            .unwrap_or(false)
852    }
853
854    /// Records a repository's visibility as GitHub gave it at `at` (ms), in place of the one recorded before.
855    pub fn visibility(&self, repository: String, visibility: api::Visibility, at: u64) {
856        self.send(Write::Visibility {
857            repository,
858            visibility,
859            at,
860        });
861    }
862
863    /// Records a change auto sizing is about to write, as pending, before the file is, synced to disk. The future
864    /// has its row once it committed, `None` if it did not: then nothing may be written.
865    pub fn sizing_intent(
866        &self,
867        change: api::SizingChange,
868    ) -> impl Future<Output = Option<i64>> + use<> {
869        let (done, committed) = oneshot::channel();
870        self.send(Write::SizingIntent { change, done });
871        async move { committed.await.ok().flatten() }
872    }
873
874    /// Records what became of a pending change, synced to disk; a done one is announced as `sizing.changed` in the
875    /// same transaction. The future says whether it committed.
876    pub fn sizing_resolved(
877        &self,
878        id: i64,
879        resolution: Resolution,
880    ) -> impl Future<Output = bool> + use<> {
881        let (done, committed) = oneshot::channel();
882        self.send(Write::SizingResolved {
883            id,
884            resolution,
885            done,
886        });
887        async move { committed.await.unwrap_or(false) }
888    }
889
890    /// Records what the notifier delivered to a target: sets `set`'s keys and removes `remove`'s, in one
891    /// transaction. The future says whether it committed.
892    pub fn notify_record(
893        &self,
894        target: &str,
895        set: Vec<(String, u64)>,
896        remove: Vec<String>,
897    ) -> impl Future<Output = bool> + use<> {
898        let (done, committed) = oneshot::channel();
899        self.send(Write::Notify {
900            change: NotifyChange::Record {
901                target: target.to_owned(),
902                set,
903                remove,
904            },
905            done,
906        });
907        async move { committed.await.unwrap_or(false) }
908    }
909
910    /// Forgets what the notifier kept for every target but `targets`. The future says whether it committed.
911    pub fn notify_keep(&self, targets: Vec<String>) -> impl Future<Output = bool> + use<> {
912        let (done, committed) = oneshot::channel();
913        self.send(Write::Notify {
914            change: NotifyChange::Keep(targets),
915            done,
916        });
917        async move { committed.await.unwrap_or(false) }
918    }
919
920    /// Deletes what is older than `history` keeps, at `now` (ms), off the writer, in batches the writer's
921    /// transactions fit between. A prune while the last one runs does nothing. The handle finishes with it; dropping
922    /// it lets it run.
923    pub fn prune(&self, now: u64, history: &History) -> tokio::task::JoinHandle<()> {
924        let pruner = self.pruner.clone();
925        let closed = self.closed.clone();
926        let history = history.clone();
927        tokio::task::spawn_blocking(move || {
928            let db = match pruner.try_lock() {
929                Ok(db) => db,
930                Err(TryLockError::Poisoned(poisoned)) => poisoned.into_inner(),
931                Err(TryLockError::WouldBlock) => return,
932            };
933            let db = prune::Db {
934                connection: &db,
935                stop: &closed,
936            };
937            if !db.stop.load(Ordering::Relaxed) {
938                prune::run(&db, now, &history);
939            }
940        })
941    }
942
943    /// Stops a running prune after its batch, and any later one before it starts, so a shutdown does not wait for
944    /// one. What a stopped prune left, the next daemon's deletes. Writes go on as before.
945    pub fn close(&self) {
946        self.closed.store(true, Ordering::Relaxed);
947    }
948
949    /// Waits until every write sent before is done.
950    pub async fn flush(&self) {
951        let (done, wait) = oneshot::channel();
952        self.send(Write::Flush(done));
953        let _ = wait.await;
954    }
955
956    /// Every event from now on, durable or live. Subscribe before reading what was stored, then skip what the read
957    /// returned, and nothing is missed or sent twice.
958    pub fn subscribe(&self) -> broadcast::Receiver<Arc<EventEnvelope>> {
959        self.events.subscribe()
960    }
961
962    async fn read<T: Send + 'static>(
963        &self,
964        query: impl FnOnce(&Connection) -> rusqlite::Result<T> + Send + 'static,
965    ) -> Result<T, StoreError> {
966        let reader = self.reader.clone();
967        // A panic during a read leaves the connection usable: reads change nothing.
968        Ok(tokio::task::spawn_blocking(move || query(&lock(&reader))).await??)
969    }
970
971    /// One page of jobs as `query` asks, which [`api::JobQuery::check`] passed (ADR 0135).
972    pub async fn jobs(&self, query: api::JobQuery) -> Result<api::JobList, StoreError> {
973        self.read(move |db| read::jobs(db, &query)).await
974    }
975
976    /// What each slot ran over `window` (ADR 0135), with the host samples not yet stored counted as watched time.
977    pub async fn timeline(&self, window: api::Span) -> Result<api::SlotTimeline, StoreError> {
978        let held: Vec<u64> = lock(&self.held).iter().map(|sample| sample.at).collect();
979        self.read(move |db| lens::timeline(db, window, &held, api::SAMPLE_INTERVAL))
980            .await
981    }
982
983    /// Job `id` beside the job `with` names, or its baseline (ADR 0135); `None` without either job.
984    pub async fn compare(
985        &self,
986        id: u64,
987        with: Option<u64>,
988    ) -> Result<Option<api::JobComparison>, StoreError> {
989        self.read(move |db| lens::compare(db, id, with)).await
990    }
991
992    /// A job with its steps.
993    pub async fn job(&self, id: u64) -> Result<Option<api::Job>, StoreError> {
994        self.read(move |db| read::job(db, id)).await
995    }
996
997    /// Up to `limit` other jobs of `repository`'s run `run` than `except`, newest attempt first (ADR 0134).
998    pub async fn run_jobs(
999        &self,
1000        repository: &str,
1001        run: u64,
1002        except: u64,
1003        limit: usize,
1004    ) -> Result<RunJobs, StoreError> {
1005        let repository = repository.to_owned();
1006        self.read(move |db| run::run_jobs(db, &repository, run, except, limit))
1007            .await
1008    }
1009
1010    /// A job's samples, oldest first; `None` if there is no such job.
1011    pub async fn job_samples(
1012        &self,
1013        id: u64,
1014        interval: Duration,
1015    ) -> Result<Option<api::SampleSeries>, StoreError> {
1016        self.read(move |db| read::job_samples(db, id, interval))
1017            .await
1018    }
1019
1020    /// Up to `limit` log lines of a job from line `from`, with their numbers.
1021    pub async fn log_lines(
1022        &self,
1023        job: u64,
1024        from: u64,
1025        limit: u64,
1026    ) -> Result<Vec<(u64, String)>, StoreError> {
1027        self.read(move |db| read::log_lines(db, job, from, limit))
1028            .await
1029    }
1030
1031    /// A job's log as structure (ADR 0133); `None` without the job.
1032    pub async fn log_outline(&self, job: u64) -> Result<Option<api::LogOutline>, StoreError> {
1033        self.read(move |db| outline::outline(db, job)).await
1034    }
1035
1036    /// Up to `limit` styled lines of a job's log from line `from`, fewer once they hold 4 MiB of text; each shows at most
1037    /// [`api::LINE_BYTES_MAX`] bytes of its line.
1038    pub async fn styled_lines(
1039        &self,
1040        job: u64,
1041        from: u64,
1042        limit: u64,
1043    ) -> Result<Vec<api::StyledLine>, StoreError> {
1044        self.read(move |db| outline::styled_lines(db, job, from, limit))
1045            .await
1046    }
1047
1048    /// One page of a search of a job's log, each batch of lines its own read, so the search never holds the reader
1049    /// for a whole log.
1050    pub async fn search_log(
1051        &self,
1052        job: u64,
1053        mut search: api::Search,
1054    ) -> Result<api::LogSearch, StoreError> {
1055        // Each batch is styled and marked where it is read, off the async workers.
1056        while let Some(want) = search.wants() {
1057            search = self
1058                .read(move |db| {
1059                    search.read(read::lines_from(db, job, want)?);
1060                    Ok(search)
1061                })
1062                .await?;
1063        }
1064        Ok(search.finish())
1065    }
1066
1067    /// Up to `limit` durable events after `after`, oldest first.
1068    pub async fn events_after(
1069        &self,
1070        after: u64,
1071        limit: u64,
1072    ) -> Result<Vec<EventEnvelope>, StoreError> {
1073        self.read(move |db| read::events_after(db, after, limit))
1074            .await
1075    }
1076
1077    /// The changes auto sizing wrote since `since` (ms), oldest first.
1078    pub async fn sizing_changes(&self, since: u64) -> Result<Vec<api::SizingChange>, StoreError> {
1079        self.read(move |db| read::sizing_changes(db, since)).await
1080    }
1081
1082    /// When each profile last changed, and the changes still pending.
1083    pub async fn auto_sizing(&self) -> Result<AutoSizing, StoreError> {
1084        self.read(read::auto_sizing).await
1085    }
1086
1087    /// The profiles and repositories as the stored events last announced them, so a restarted daemon announces
1088    /// only what changed since, including what its config no longer has. Blocks on the database: for startup.
1089    pub fn announced(&self) -> Result<Announced, StoreError> {
1090        Ok(read::announced(&lock(&self.reader))?)
1091    }
1092
1093    /// The repositories recorded paused, for the engine to start with. Blocks on the database: for startup.
1094    pub fn paused_repositories(&self) -> Result<Vec<String>, StoreError> {
1095        Ok(read::paused(&lock(&self.reader))?)
1096    }
1097
1098    /// The slice's samples since `since` (seconds), oldest first, the ones not yet stored included.
1099    pub async fn host_samples(&self, since: u64) -> Result<Vec<api::HostSample>, StoreError> {
1100        // Copied before the read: the writer removes a sample only once it is stored, so each one is in the copy,
1101        // the read, or both.
1102        let held = lock(&self.held).clone();
1103        let mut samples = self.read(move |db| read::host_samples(db, since)).await?;
1104        let stored = samples.last().map_or(0, |sample| sample.at);
1105        samples.extend(
1106            held.into_iter()
1107                .filter(|sample| sample.at >= since && sample.at > stored),
1108        );
1109        Ok(samples)
1110    }
1111
1112    /// The jobs runnerd's runners started and finished from `since` until `now` (seconds).
1113    pub async fn activity(&self, since: u64, now: u64) -> Result<api::Activity, StoreError> {
1114        self.read(move |db| read::activity(db, since, now)).await
1115    }
1116
1117    /// Finished jobs per repository and bucket.
1118    pub async fn stats(&self, query: api::StatsQuery, now: u64) -> Result<api::Stats, StoreError> {
1119        self.read(move |db| read::stats(db, &query, now)).await
1120    }
1121
1122    /// What sizing reads: the finished jobs queued at or after `since` (ms) and the runners since.
1123    pub async fn sizing(&self, since: u64) -> Result<sizing::History, StoreError> {
1124        self.read(move |db| read::sizing(db, since)).await
1125    }
1126
1127    /// What pricing at GitHub's rates reads: the completed and lost jobs dated at or after `since` (ms), each
1128    /// repository's visibility and since when the history holds every final job.
1129    pub async fn pricing(&self, since: u64) -> Result<api::PricingHistory, StoreError> {
1130        self.read(move |db| read::pricing(db, since)).await
1131    }
1132
1133    /// The jobs completed or lost that were queued at or after `since` (ms), as insights read them, from when the
1134    /// history holds every job, and the queued jobs the last plan had waiting for the host's memory.
1135    pub async fn insights(&self, since: u64) -> Result<crate::insights::History, StoreError> {
1136        let memory = self.latest_host_memory();
1137        let (waiting, seen) = lock(&self.host_waiting).clone();
1138        self.read(move |db| {
1139            let mut history = read::insights(db, since)?;
1140            if let Some(memory) = memory.filter(|_| !waiting.is_empty()) {
1141                let jobs = read::held_jobs(db, &waiting)?;
1142                history.host =
1143                    (!jobs.is_empty()).then_some(crate::insights::HostHold { memory, seen, jobs });
1144            }
1145            Ok(history)
1146        })
1147        .await
1148    }
1149
1150    /// The newest durable event's ID; 0 before the first.
1151    pub async fn last_event(&self) -> Result<u64, StoreError> {
1152        self.read(read::last_event).await
1153    }
1154
1155    /// What the notifier kept for each target, by target, then key.
1156    pub async fn notify_state(
1157        &self,
1158    ) -> Result<BTreeMap<String, BTreeMap<String, u64>>, StoreError> {
1159        self.read(read::notify_state).await
1160    }
1161
1162    /// The jobs queued at or after `since` (ms), whatever became of them, as warm runners learn from them.
1163    pub async fn warm(&self, since: u64) -> Result<Vec<warm::Seen>, StoreError> {
1164        self.read(move |db| read::warm(db, since)).await
1165    }
1166
1167    /// The last measurement of every volume, with its time (seconds), by name.
1168    pub async fn cache_sizes_now(&self) -> Result<Measured, StoreError> {
1169        self.read(read::latest_cache_sizes).await
1170    }
1171
1172    /// What the build cache served and took from the runners started since `since` (ms), summed.
1173    pub async fn build_cache_since(&self, since: u64) -> Result<api::BuildCacheCounts, StoreError> {
1174        self.read(move |db| read::build_cache_since(db, since))
1175            .await
1176    }
1177
1178    /// Every volume measurement since `since` (seconds).
1179    pub async fn cache_history(&self, since: u64) -> Result<Vec<api::CacheSize>, StoreError> {
1180        self.read(move |db| read::cache_history(db, since)).await
1181    }
1182
1183    /// The jobs not yet closed, for the engine after a restart.
1184    pub async fn open_jobs(&self) -> Result<Vec<OpenJob>, StoreError> {
1185        self.read(read::open_jobs).await
1186    }
1187
1188    /// The newest named job each slot ran, for the engine after a restart.
1189    pub async fn slot_jobs(&self) -> Result<Vec<SlotJob>, StoreError> {
1190        self.read(read::slot_jobs).await
1191    }
1192
1193    /// The latest second a runner was named for (`runner_named`), or any runner name starting with `prefix` carries,
1194    /// of a runner or of a job's runner: the engine names runners later after a restart, so no name it used before
1195    /// is used twice.
1196    pub async fn last_runner_second(&self, prefix: String) -> Result<Option<u64>, StoreError> {
1197        self.read(move |db| read::last_runner_second(db, &prefix))
1198            .await
1199    }
1200
1201    /// The job a runner took, if the history knows it.
1202    pub async fn runner_job(&self, runner: &str) -> Result<Option<u64>, StoreError> {
1203        let runner = runner.to_owned();
1204        self.read(move |db| read::runner_job(db, &runner)).await
1205    }
1206
1207    /// How a runner was admitted, with what it reserves since, if the history knows it.
1208    pub async fn runner_admitted(
1209        &self,
1210        runner: &str,
1211    ) -> Result<Option<api::Admission>, StoreError> {
1212        let runner = runner.to_owned();
1213        self.read(move |db| read::runner_admission(db, &runner))
1214            .await
1215    }
1216
1217    /// When a forced pause that cancelled a runner's run removes it if it still runs, in ms since the epoch, if one
1218    /// did.
1219    pub async fn runner_cancel_deadline(&self, runner: &str) -> Result<Option<u64>, StoreError> {
1220        let runner = runner.to_owned();
1221        self.read(move |db| read::runner_cancel_deadline(db, &runner))
1222            .await
1223    }
1224
1225    /// The memory cap a runner started with, as its record kept it, for when Docker cannot tell.
1226    pub async fn runner_memory_cap(&self, runner: &str) -> Result<Option<u64>, StoreError> {
1227        let runner = runner.to_owned();
1228        self.read(move |db| read::runner_memory_cap(db, &runner))
1229            .await
1230    }
1231
1232    /// Where a job's log capture continues.
1233    pub async fn log_position(&self, job: u64) -> Result<LogPosition, StoreError> {
1234        self.read(move |db| read::log_position(db, job)).await
1235    }
1236
1237    /// How far the mirror's log was counted.
1238    pub async fn mirror_log_until(&self) -> Result<MirrorPosition, StoreError> {
1239        self.read(read::mirror_log_until).await
1240    }
1241}
1242
1243/// What sizing reads from the database at `path`, opened read-only and left as it is, as `runnerd doctor` reads it
1244/// while no daemon runs; `None` when there is no database yet. In a directory it may write, SQLite keeps the
1245/// `-wal` and `-shm` files a read of a WAL database needs next to it; in one it may not, it reads the file as it is,
1246/// or refuses when the `-wal` holds changes the file does not have yet.
1247pub fn sizing_at(path: &Path, since: u64) -> Result<Option<sizing::History>, StoreError> {
1248    read_only(path, |db| sizing_of(db, since))
1249}
1250
1251/// How the jobs of the history at `path` that started from `since` (ms) ran, read as [`sizing_at`] reads, for a replay
1252/// that runs them so (`admission::simulate`): their samples, when the daemon could name each, what the host's programs
1253/// held. `overhead` is what the slice holds beside its runners, in bytes; `None` measures it from the history's busy
1254/// moments, as `read::trace` says. `named_by` is the longest a known job waits to be read: the poll interval.
1255pub fn trace_at(
1256    path: &Path,
1257    since: u64,
1258    named_by: u64,
1259    overhead: Option<u64>,
1260) -> Result<Option<Trace>, StoreError> {
1261    read_only(path, |db| {
1262        if version(db)? == 0 {
1263            return Ok(Trace::default());
1264        }
1265        Ok(read::trace(db, since, named_by, overhead)?)
1266    })
1267}
1268
1269/// When the last job of the history at `path` was queued, in ms, read as [`sizing_at`] reads; `None` with no job yet.
1270/// A replay's window counts back from it, so a backup replays its own last hours.
1271pub fn last_queued_at(path: &Path) -> Result<Option<u64>, StoreError> {
1272    let last = read_only(path, |db| {
1273        if version(db)? == 0 {
1274            return Ok(None);
1275        }
1276        Ok(db.query_row("SELECT MAX(queued_at) FROM jobs", [], |row| row.get(0))?)
1277    })?;
1278    Ok(last.flatten())
1279}
1280
1281/// What `read` finds in the database at `path`, opened read-only and left as it is; `None` when there is none.
1282fn read_only<T>(
1283    path: &Path,
1284    read: impl Fn(&Connection) -> Result<T, StoreError>,
1285) -> Result<Option<T>, StoreError> {
1286    if !path.exists() {
1287        return Ok(None);
1288    }
1289    let read = |path: &str, flags| -> Result<T, StoreError> {
1290        let db = Connection::open_with_flags(path, flags)?;
1291        db.busy_timeout(Duration::from_secs(5))?;
1292        read(&db)
1293    };
1294    let name = path.to_string_lossy();
1295    match read(&name, OpenFlags::SQLITE_OPEN_READ_ONLY) {
1296        Err(StoreError::Sqlite(error))
1297            if matches!(
1298                error.sqlite_error_code(),
1299                Some(rusqlite::ErrorCode::ReadOnly | rusqlite::ErrorCode::CannotOpen)
1300            ) =>
1301        {
1302            // No `-shm` can be made: read the file alone, unless the `-wal` holds what it lacks.
1303            let mut wal = path.as_os_str().to_owned();
1304            wal.push("-wal");
1305            let wal = PathBuf::from(wal);
1306            if std::fs::metadata(&wal).is_ok_and(|wal| wal.len() > 0) {
1307                return Err(StoreError::Unread(wal));
1308            }
1309            let uri = format!(
1310                "file:{}?mode=ro&immutable=1",
1311                name.replace('%', "%25")
1312                    .replace('?', "%3f")
1313                    .replace('#', "%23")
1314            );
1315            read(
1316                &uri,
1317                OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_URI,
1318            )
1319        }
1320        history => history,
1321    }
1322    .map(Some)
1323}
1324
1325/// What sizing reads from the database, without creating its schema; an older one reads as [`VERSION`].
1326fn sizing_of(db: &Connection, since: u64) -> Result<sizing::History, StoreError> {
1327    match version(db)? {
1328        0 => return Ok(sizing::History::default()),
1329        VERSION => {}
1330        _ => read_as_current(db)?,
1331    }
1332    let mut history = read::sizing(db, since)?;
1333    let auto = read::auto_sizing(db)?;
1334    history.memory_set = auto.memory;
1335    history.slots_set = auto.slots;
1336    history.changes = read::sizing_changes(db, since.saturating_sub(sizing::memory::HISTORY))?;
1337    Ok(history)
1338}
1339
1340/// The database's schema version: 0 for a new one, else from [`OLDEST`] to [`VERSION`]; any other is refused.
1341fn version(db: &Connection) -> Result<u32, StoreError> {
1342    let found: u32 = db.pragma_query_value(None, "user_version", |row| row.get(0))?;
1343    match found {
1344        0 | OLDEST..=VERSION => Ok(found),
1345        found if found > VERSION => Err(StoreError::Newer {
1346            found,
1347            supported: VERSION,
1348        }),
1349        found => Err(StoreError::Older {
1350            found,
1351            oldest: OLDEST,
1352        }),
1353    }
1354}
1355
1356/// Lets a read-only look at a database from before [`VERSION`], which it cannot migrate, read it as this build
1357/// does: a table it lacks becomes an empty temporary one, and a table that lacks columns a temporary view of it
1358/// that adds them, each with its default. Temporary objects come first in name lookups.
1359fn read_as_current(db: &Connection) -> Result<(), StoreError> {
1360    let current = Connection::open_in_memory()?;
1361    current.execute_batch(SCHEMA)?;
1362    let tables: Vec<(String, String)> = current
1363        .prepare(
1364            "SELECT name, sql FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'",
1365        )?
1366        .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?
1367        .collect::<rusqlite::Result<_>>()?;
1368    let columns =
1369        |db: &Connection, table: &str| -> rusqlite::Result<Vec<(String, Option<String>)>> {
1370            db.prepare("SELECT name, dflt_value FROM pragma_table_info(?1, 'main')")?
1371                .query_map([table], |row| Ok((row.get(0)?, row.get(1)?)))?
1372                .collect()
1373        };
1374    for (table, sql) in tables {
1375        let found: Vec<String> = columns(db, &table)?
1376            .into_iter()
1377            .map(|(name, _)| name)
1378            .collect();
1379        if found.is_empty() {
1380            db.execute_batch(&sql.replacen("CREATE TABLE", "CREATE TEMP TABLE", 1))?;
1381            continue;
1382        }
1383        let added: Vec<String> = columns(&current, &table)?
1384            .into_iter()
1385            .filter(|(name, _)| !found.contains(name))
1386            .map(|(name, default)| format!("{} AS {name}", default.as_deref().unwrap_or("NULL")))
1387            .collect();
1388        if !added.is_empty() {
1389            db.execute_batch(&format!(
1390                "CREATE TEMP VIEW {table} AS SELECT *, {} FROM main.{table}",
1391                added.join(", ")
1392            ))?;
1393        }
1394    }
1395    Ok(())
1396}
1397
1398/// WAL, so the API reads while the writer writes; a busy timeout instead of an immediate error.
1399fn configure(db: &Connection) -> rusqlite::Result<()> {
1400    db.pragma_update(None, "journal_mode", "wal")?;
1401    db.pragma_update(None, "synchronous", "normal")?;
1402    db.pragma_update(None, "foreign_keys", "on")?;
1403    db.busy_timeout(Duration::from_secs(5))
1404}
1405
1406/// Creates the schema in a new database, in one transaction with its version, or runs the migrations an older one
1407/// lacks, each in its own transaction with the version it brings it to.
1408fn prepare(db: &mut Connection) -> Result<(), StoreError> {
1409    let found = version(db)?;
1410    if found == 0 {
1411        let transaction = db.transaction()?;
1412        transaction.execute_batch(SCHEMA)?;
1413        transaction.pragma_update(None, "user_version", VERSION)?;
1414        transaction.commit()?;
1415        return Ok(());
1416    }
1417    for (version, migration) in MIGRATIONS.iter().filter(|(version, _)| *version > found) {
1418        let transaction = db.transaction()?;
1419        transaction.execute_batch(migration)?;
1420        transaction.pragma_update(None, "user_version", version)?;
1421        transaction.commit()?;
1422    }
1423    Ok(())
1424}
1425
1426/// The time before this daemon when no runnerd ran, in ms: from the history's last sign of life, its newest host
1427/// sample, event or open, to this daemon's start. Host samples are stored a minute's worth at a time, so a crash
1428/// may count up to a minute more. Without a sign of life, as in a new history, there is none: the history's start
1429/// clips the waits of jobs queued before it (`history_kept`).
1430#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1431struct Downtime {
1432    from: u64,
1433    until: u64,
1434}
1435
1436impl Downtime {
1437    /// The downtime before a daemon that starts at `now`; the jobs still queued in the history leave it out of
1438    /// their waits, as the ones first seen after do when they are recorded. The open is a sign of life itself, kept
1439    /// with them: a start that fails after it, and is retried, does not hand out the same downtime again.
1440    fn before(db: &mut Connection, now: u64) -> Result<Option<Downtime>, StoreError> {
1441        // Immediate: it reads, then writes. A deferred transaction that has read cannot wait for the write lock, and
1442        // fails at once while another connection, such as a dropped store's writer finishing, holds it.
1443        let transaction = db.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?;
1444        let last: Option<u64> = transaction.query_row(
1445            "SELECT max(at) FROM (SELECT max(at) AS at FROM host_samples UNION ALL SELECT max(at) FROM events \
1446             UNION ALL SELECT value FROM meta WHERE key = 'opened_at')",
1447            [],
1448            |row| row.get(0),
1449        )?;
1450        let downtime = last
1451            .filter(|last| *last < now)
1452            .map(|from| Downtime { from, until: now });
1453        if let Some(downtime) = downtime {
1454            let queued: Vec<(u64, u64)> = transaction
1455                .prepare("SELECT id, queued_at FROM jobs WHERE state = 'queued'")?
1456                .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?
1457                .collect::<rusqlite::Result<_>>()?;
1458            for (job, queued_at) in queued {
1459                transaction.execute(
1460                    "UPDATE jobs SET unseen = unseen + ?2 WHERE id = ?1",
1461                    rusqlite::params![job, downtime.of(queued_at)],
1462                )?;
1463            }
1464        }
1465        transaction.execute(
1466            "INSERT OR REPLACE INTO meta (key, value) VALUES ('opened_at', ?1)",
1467            [now],
1468        )?;
1469        transaction.commit()?;
1470        Ok(downtime)
1471    }
1472
1473    /// How much of it a job queued at `queued_at` (ms) waited through.
1474    fn of(self, queued_at: u64) -> u64 {
1475        self.until.saturating_sub(queued_at.max(self.from))
1476    }
1477
1478    /// How much of what [`Downtime::of`] gave a job queued at `queued_at` came after it started at `started_at`
1479    /// (ms), as for a job a runner that outlived the last daemon took while none ran.
1480    fn after(self, queued_at: u64, started_at: u64) -> u64 {
1481        self.of(queued_at)
1482            .saturating_sub(started_at.saturating_sub(queued_at.max(self.from)))
1483    }
1484}
1485
1486/// Logs a failed write; the writer goes on with the next one.
1487fn logged<T>(what: &str, result: rusqlite::Result<T>) -> Option<T> {
1488    result
1489        .map_err(|error| warn!("job history: cannot {what}: {error}"))
1490        .ok()
1491}
1492
1493#[cfg(test)]
1494mod tests;