use std::collections::{BTreeMap, BTreeSet};
use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex, TryLockError};
use std::time::Duration;
use rusqlite::{Connection, OpenFlags};
use tokio::sync::{OwnedSemaphorePermit, Semaphore, broadcast, mpsc, oneshot};
use tracing::warn;
use crate::admission::trace::Trace;
use crate::api::{self, Event, EventEnvelope};
use crate::config::History;
use crate::github::{JobDetails, NamedRun, QueuedJob};
use crate::observe::disk::{Window, Written};
use crate::observe::metrics::{CpuHeld, Events};
use crate::scheduler::warm;
use crate::sizing;
use crate::sync::lock;
mod lens;
mod outline;
mod prune;
mod read;
mod run;
mod write;
pub use run::RunJobs;
const SCHEMA: &str = include_str!("store/schema.sql");
const OLDEST: u32 = 15;
const MIGRATIONS: &[(u32, &str)] = &[
(
16,
"ALTER TABLE jobs ADD COLUMN unseen INTEGER NOT NULL DEFAULT 0;",
),
(
17,
"ALTER TABLE jobs ADD COLUMN cpu_periods INTEGER;\n\
ALTER TABLE jobs ADD COLUMN cpu_throttled INTEGER;\n\
ALTER TABLE jobs ADD COLUMN cpu_stalled INTEGER;\n\
ALTER TABLE samples ADD COLUMN cpu_periods INTEGER;\n\
ALTER TABLE samples ADD COLUMN cpu_throttled INTEGER;\n\
ALTER TABLE samples ADD COLUMN cpu_stalled INTEGER;",
),
(
18,
"ALTER TABLE jobs ADD COLUMN slot_waited INTEGER NOT NULL DEFAULT 0;",
),
(
19,
"ALTER TABLE jobs ADD COLUMN seen_at INTEGER;\n\
ALTER TABLE runners ADD COLUMN purpose TEXT;\n\
ALTER TABLE runners ADD COLUMN stopped_for TEXT;",
),
(
20,
"ALTER TABLE jobs ADD COLUMN runner_fault INTEGER NOT NULL DEFAULT 0;",
),
(21, "ALTER TABLE runners ADD COLUMN cancel_by INTEGER;"),
(
22,
"CREATE TABLE log_outlines (job INTEGER PRIMARY KEY NOT NULL, from_line INTEGER NOT NULL, \
errors INTEGER NOT NULL, warnings INTEGER NOT NULL, groups_kept INTEGER NOT NULL, \
marks_kept INTEGER NOT NULL, sections_kept INTEGER NOT NULL, dropped INTEGER NOT NULL, \
hidden INTEGER NOT NULL);\n\
CREATE TABLE log_groups (job INTEGER NOT NULL, start INTEGER NOT NULL, end_line INTEGER, \
depth INTEGER NOT NULL, title TEXT NOT NULL, PRIMARY KEY (job, start)) WITHOUT ROWID;\n\
CREATE INDEX log_groups_open ON log_groups (job, start) WHERE end_line IS NULL;\n\
CREATE TABLE log_marks (job INTEGER NOT NULL, line INTEGER NOT NULL, kind TEXT NOT NULL, \
PRIMARY KEY (job, line)) WITHOUT ROWID;",
),
(
23,
"CREATE TABLE sessions (id INTEGER PRIMARY KEY, started_at INTEGER NOT NULL, sampled_at INTEGER);\n\
ALTER TABLE records ADD COLUMN gone_at INTEGER;",
),
(
24,
"ALTER TABLE jobs ADD COLUMN disk_written INTEGER;\n\
ALTER TABLE jobs ADD COLUMN disk_write_ios INTEGER;\n\
ALTER TABLE jobs ADD COLUMN disk_dirty INTEGER;\n\
ALTER TABLE jobs ADD COLUMN disk_measure TEXT;\n\
ALTER TABLE runners ADD COLUMN cgroup_gone_at INTEGER;\n\
ALTER TABLE samples ADD COLUMN written INTEGER;\n\
ALTER TABLE samples ADD COLUMN write_ios INTEGER;\n\
ALTER TABLE samples ADD COLUMN dirty INTEGER;\n\
CREATE TABLE disk_windows (start INTEGER PRIMARY KEY NOT NULL, end_at INTEGER NOT NULL, \
device TEXT NOT NULL, store TEXT, drive INTEGER, slice INTEGER, daemon INTEGER, commits INTEGER) WITHOUT ROWID;\n\
CREATE INDEX jobs_by_runner ON jobs (runner);",
),
];
const VERSION: u32 = match MIGRATIONS.last() {
Some((version, _)) => *version,
None => OLDEST,
};
pub(crate) const LOG_BACKLOG: u64 = 8 << 20;
const HOST_SAMPLE_BATCH: usize = 30;
const EVENT_BUFFER: usize = 4096;
#[derive(Debug, thiserror::Error)]
pub enum StoreError {
#[error("{0}")]
Sqlite(#[from] rusqlite::Error),
#[error(
"the job history database is version {found}, newer than the {supported} this runnerd reads; \
run a newer runnerd or move the database aside"
)]
Newer {
found: u32,
supported: u32,
},
#[error(
"the job history database is version {found}, from a runnerd before 0.0.19; this runnerd reads versions \
from {oldest} on: move the database aside"
)]
Older {
found: u32,
oldest: u32,
},
#[error("{0}")]
Io(#[from] std::io::Error),
#[error("a database read stopped: {0}")]
Join(#[from] tokio::task::JoinError),
#[error(
"{} holds changes not yet in the job history, which this user cannot read; run runnerd doctor as the user \
the daemon runs as",
.0.display()
)]
Unread(PathBuf),
}
impl StoreError {
pub fn is_shutdown(&self) -> bool {
matches!(self, StoreError::Join(join) if join.is_cancelled())
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RunnerRecord {
pub name: String,
pub repository: String,
pub profile: String,
pub slot: usize,
pub container: String,
pub started_at: u64,
pub memory_cap: Option<u64>,
pub cpu_cap: Option<u64>,
pub admission: Option<api::Admission>,
pub purpose: Option<&'static str>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SampleRecord {
pub runner: String,
pub at: u64,
pub cpu: u64,
pub cpu_usage: u64,
pub memory: u64,
pub network_received: u64,
pub network_sent: u64,
pub events: Events,
pub cpu_held: CpuHeld,
pub written: Option<Written>,
pub dirty: Option<u64>,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct Measured {
pub at: Option<u64>,
pub bytes: BTreeMap<String, u64>,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct Announced {
pub profiles: Vec<api::ProfileStatus>,
pub repositories: Vec<api::RepositoryStatus>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct OpenJob {
pub id: u64,
pub repository: String,
pub state: api::JobState,
pub runner: Option<String>,
pub runner_removed_at: Option<u64>,
pub run_id: u64,
pub run_attempt: u32,
pub workflow: Option<String>,
pub queued_at: u64,
pub labels: Vec<String>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SlotJob {
pub profile: String,
pub slot: usize,
pub key: crate::admission::Key,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct LogPosition {
pub page: u64,
pub offset: u64,
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct MirrorPosition {
pub at: u64,
pub seen: u64,
}
#[derive(Debug)]
enum Write {
Queued {
repository: String,
job: QueuedJob,
seen_at: u64,
},
RunnerStarted(RunnerRecord),
RunnerAdmission {
runner: String,
admission: api::Admission,
},
RunnerOomKilled(String),
RunnerStoppedFor {
runner: String,
reason: &'static str,
},
RunnerCancelBy {
runner: String,
at: u64,
},
HostWaited {
jobs: Vec<u64>,
ms: u64,
},
SlotWaited {
jobs: Vec<u64>,
ms: u64,
},
BuildCache {
runner: String,
counts: api::BuildCacheCounts,
},
RunnerRemoved {
runner: String,
at: u64,
},
RunnersGone {
running: Vec<String>,
at: u64,
},
Claimed {
job: u64,
run_id: u64,
runner: String,
at: u64,
},
Details {
details: JobDetails,
},
WorkflowNamed(NamedRun),
Elsewhere {
details: JobDetails,
},
Lost {
job: u64,
at: u64,
},
Unanswered {
job: u64,
at: u64,
},
Log {
job: u64,
lines: Vec<String>,
position: Option<LogPosition>,
max_bytes: u64,
_backlog: OwnedSemaphorePermit,
},
Samples(Vec<SampleRecord>),
CgroupsGone {
runners: Vec<String>,
at: u64,
},
DiskWindow(Window),
HostSample(api::HostSample),
MirrorBytes {
bytes: Vec<(String, u64)>,
until: MirrorPosition,
done: oneshot::Sender<bool>,
},
CacheSizes {
at: u64,
sizes: BTreeMap<String, u64>,
},
Event(Box<Event>),
RecordMade(api::Record),
RecordSkipped {
job: u64,
why: String,
},
RecordRemoved {
job: u64,
at: u64,
by: api::RecordRemoval,
},
RecordsListed {
jobs: Vec<u64>,
at: u64,
},
Paused {
repositories: Vec<String>,
done: oneshot::Sender<bool>,
},
Visibility {
repository: String,
visibility: api::Visibility,
at: u64,
},
SizingIntent {
change: api::SizingChange,
done: oneshot::Sender<Option<i64>>,
},
SizingResolved {
id: i64,
resolution: Resolution,
done: oneshot::Sender<bool>,
},
Notify {
change: NotifyChange,
done: oneshot::Sender<bool>,
},
RunnerNamed {
secs: u64,
done: oneshot::Sender<bool>,
},
Flush(oneshot::Sender<()>),
}
#[derive(Debug)]
enum NotifyChange {
Record {
target: String,
set: Vec<(String, u64)>,
remove: Vec<String>,
},
Keep(Vec<String>),
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Resolution {
Done,
Failed,
Uncertain,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct AutoSizing {
pub last: BTreeMap<String, u64>,
pub pending: Vec<(i64, api::SizingChange)>,
pub cpus: Vec<sizing::CpusSet>,
pub memory: Vec<sizing::MemorySet>,
pub slots: Vec<sizing::SlotsSet>,
}
#[derive(Debug, Clone)]
pub struct Store {
writes: mpsc::UnboundedSender<Write>,
reader: Arc<Mutex<Connection>>,
pruner: Arc<Mutex<Connection>>,
closed: Arc<AtomicBool>,
held: Arc<Mutex<Vec<api::HostSample>>>,
latest_host: Arc<Mutex<Option<api::HostSample>>>,
latest_host_memory: Arc<Mutex<Option<api::HostMemory>>>,
host_waiting: Arc<Mutex<(BTreeSet<u64>, u64)>>,
latest_samples: Arc<Mutex<BTreeMap<String, SampleRecord>>>,
gone: Arc<Mutex<BTreeSet<String>>>,
events: broadcast::Sender<Arc<EventEnvelope>>,
backlog: Arc<Semaphore>,
}
impl Store {
pub fn open(path: &Path) -> Result<Store, StoreError> {
Store::open_at(path, crate::time::now_ms())
}
fn open_at(path: &Path, now: u64) -> Result<Store, StoreError> {
if let Some(dir) = path.parent().filter(|dir| !dir.as_os_str().is_empty()) {
std::fs::DirBuilder::new()
.recursive(true)
.mode(0o700)
.create(dir)?;
}
std::fs::OpenOptions::new()
.create(true)
.append(true)
.mode(0o600)
.open(path)?;
let mut db = Connection::open(path)?;
configure(&db)?;
prepare(&mut db)?;
let downtime = Downtime::before(&mut db, now)?;
db.execute("INSERT INTO sessions (started_at) VALUES (?1)", [now])?;
let session = db.last_insert_rowid();
let reader = Connection::open_with_flags(
path,
OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX,
)?;
configure(&reader)?;
let pruner = Connection::open(path)?;
configure(&pruner)?;
let (events, _) = broadcast::channel(EVENT_BUFFER);
let (writes, receiver) = mpsc::unbounded_channel();
let thread_events = events.clone();
let held = Arc::new(Mutex::new(Vec::new()));
let thread_held = held.clone();
std::thread::Builder::new()
.name("runnerd-store".to_owned())
.spawn(move || {
write::run(
write::Started {
db,
downtime,
session,
},
receiver,
&thread_events,
&thread_held,
);
})?;
Ok(Store {
writes,
reader: Arc::new(Mutex::new(reader)),
pruner: Arc::new(Mutex::new(pruner)),
closed: Arc::new(AtomicBool::new(false)),
held,
latest_host: Arc::new(Mutex::new(None)),
latest_host_memory: Arc::new(Mutex::new(None)),
host_waiting: Arc::new(Mutex::new((BTreeSet::new(), 0))),
latest_samples: Arc::new(Mutex::new(BTreeMap::new())),
gone: Arc::new(Mutex::new(BTreeSet::new())),
events,
backlog: Arc::new(Semaphore::new(
usize::try_from(LOG_BACKLOG).unwrap_or(usize::MAX),
)),
})
}
fn send(&self, write: Write) {
let _ = self.writes.send(write);
}
pub fn queued(&self, repository: &str, job: &QueuedJob, seen_at: u64) {
self.send(Write::Queued {
repository: repository.to_owned(),
job: job.clone(),
seen_at,
});
}
pub fn runner_started(&self, runner: RunnerRecord) {
self.send(Write::RunnerStarted(runner));
}
pub fn runner_removed(&self, runner: &str, at: u64) {
lock(&self.latest_samples).remove(runner);
lock(&self.gone).remove(runner);
self.send(Write::RunnerRemoved {
runner: runner.to_owned(),
at,
});
}
pub fn runners_gone(&self, running: Vec<String>, at: u64) {
lock(&self.latest_samples).retain(|runner, _| running.contains(runner));
lock(&self.gone).retain(|runner| running.contains(runner));
self.send(Write::RunnersGone { running, at });
}
pub fn claimed(&self, job: u64, run_id: u64, runner: &str, at: u64) {
self.send(Write::Claimed {
job,
run_id,
runner: runner.to_owned(),
at,
});
}
pub fn details(&self, details: JobDetails) {
self.send(Write::Details { details });
}
pub fn workflow_named(&self, named: NamedRun) {
self.send(Write::WorkflowNamed(named));
}
pub fn elsewhere(&self, details: JobDetails) {
self.send(Write::Elsewhere { details });
}
pub fn unanswered(&self, job: u64, at: u64) {
self.send(Write::Unanswered { job, at });
}
pub fn lost(&self, job: u64, at: u64) {
self.send(Write::Lost { job, at });
}
pub async fn log(&self, job: u64, lines: Vec<String>, position: LogPosition, max_bytes: u64) {
let size = |line: &String| line.len() as u64 + 1;
let mut lines = lines.into_iter().map(|mut line| {
if size(&line) > LOG_BACKLOG {
let mut end = usize::try_from(LOG_BACKLOG / 2).unwrap_or(usize::MAX);
while !line.is_char_boundary(end) {
end -= 1;
}
line.truncate(end);
}
line
});
let mut next = lines.next();
loop {
let (mut part, mut bytes) = (Vec::new(), 0);
while let Some(line) = next.take_if(|line| bytes + size(line) <= LOG_BACKLOG) {
bytes += size(&line);
part.push(line);
next = lines.next();
}
let last = next.is_none();
let share = u32::try_from(bytes.max(1)).unwrap_or(u32::MAX);
let Ok(backlog) = self.backlog.clone().acquire_many_owned(share).await else {
return;
};
self.send(Write::Log {
job,
lines: part,
position: last.then_some(position),
max_bytes,
_backlog: backlog,
});
if last {
return;
}
}
}
pub fn samples(&self, samples: Vec<SampleRecord>) {
let mut latest = lock(&self.latest_samples);
for sample in &samples {
latest.insert(sample.runner.clone(), sample.clone());
}
drop(latest);
self.send(Write::Samples(samples));
}
pub fn host_sample(&self, sample: api::HostSample) {
*lock(&self.latest_host) = Some(sample.clone());
lock(&self.held).push(sample.clone());
self.send(Write::HostSample(sample));
}
pub fn latest_sample(&self, runner: &str) -> Option<SampleRecord> {
lock(&self.latest_samples).get(runner).cloned()
}
pub fn containers_gone(&self, runners: Vec<String>) {
lock(&self.gone).extend(runners);
}
pub fn containers_ended(&self, runners: Vec<String>, at: u64) {
if !runners.is_empty() {
self.send(Write::CgroupsGone { runners, at });
}
}
pub fn disk_window(&self, window: Window) {
self.send(Write::DiskWindow(window));
}
pub fn containers_read(&self, runners: &[String]) {
let mut gone = lock(&self.gone);
for runner in runners {
gone.remove(runner);
}
}
pub fn container_gone(&self, runner: &str) -> bool {
lock(&self.gone).contains(runner)
}
pub fn runner_admission(&self, runner: &str, admission: api::Admission) {
self.send(Write::RunnerAdmission {
runner: runner.to_owned(),
admission,
});
}
pub fn runner_oom_killed(&self, runner: &str) {
self.send(Write::RunnerOomKilled(runner.to_owned()));
}
pub fn runner_cancel_by(&self, runner: &str, at: u64) {
self.send(Write::RunnerCancelBy {
runner: runner.to_owned(),
at,
});
}
pub fn runner_stopped_for(&self, runner: &str, reason: &'static str) {
self.send(Write::RunnerStoppedFor {
runner: runner.to_owned(),
reason,
});
}
pub fn build_cache(&self, runner: &str, counts: api::BuildCacheCounts) {
self.send(Write::BuildCache {
runner: runner.to_owned(),
counts,
});
}
pub fn host_memory(&self, memory: api::HostMemory) {
*lock(&self.latest_host_memory) = Some(memory);
}
pub fn latest_host_memory(&self) -> Option<api::HostMemory> {
lock(&self.latest_host_memory).clone()
}
pub fn host_waiting(&self, jobs: BTreeSet<u64>, at: u64) {
*lock(&self.host_waiting) = (jobs, at);
}
pub fn host_waited(&self, jobs: Vec<u64>, ms: u64) {
self.send(Write::HostWaited { jobs, ms });
}
pub fn slot_waited(&self, jobs: Vec<u64>, ms: u64) {
self.send(Write::SlotWaited { jobs, ms });
}
pub fn latest_host_sample(&self) -> Option<api::HostSample> {
lock(&self.latest_host).clone()
}
pub async fn mirror_bytes(&self, bytes: Vec<(String, u64)>, until: MirrorPosition) -> bool {
let (done, committed) = oneshot::channel();
self.send(Write::MirrorBytes { bytes, until, done });
committed.await.unwrap_or(false)
}
pub fn cache_sizes(&self, at: u64, sizes: BTreeMap<String, u64>) {
self.send(Write::CacheSizes { at, sizes });
}
pub fn event(&self, event: Event) {
self.send(Write::Event(Box::new(event)));
}
pub fn record_made(&self, record: api::Record) {
self.send(Write::RecordMade(record));
}
pub fn record_skipped(&self, job: u64, why: String) {
self.send(Write::RecordSkipped { job, why });
}
pub fn record_removed(&self, job: u64, at: u64, by: api::RecordRemoval) {
self.send(Write::RecordRemoved { job, at, by });
}
pub fn records_listed(&self, jobs: Vec<u64>, at: u64) {
self.send(Write::RecordsListed { jobs, at });
}
pub fn paused(&self, repositories: Vec<String>) -> impl Future<Output = bool> + use<> {
let (done, committed) = oneshot::channel();
self.send(Write::Paused { repositories, done });
async move { committed.await.unwrap_or(false) }
}
pub async fn runner_named(&self, secs: u64) -> bool {
let (done, committed) = oneshot::channel();
self.send(Write::RunnerNamed { secs, done });
tokio::task::spawn_blocking(move || committed.blocking_recv().unwrap_or(false))
.await
.unwrap_or(false)
}
pub fn visibility(&self, repository: String, visibility: api::Visibility, at: u64) {
self.send(Write::Visibility {
repository,
visibility,
at,
});
}
pub fn sizing_intent(
&self,
change: api::SizingChange,
) -> impl Future<Output = Option<i64>> + use<> {
let (done, committed) = oneshot::channel();
self.send(Write::SizingIntent { change, done });
async move { committed.await.ok().flatten() }
}
pub fn sizing_resolved(
&self,
id: i64,
resolution: Resolution,
) -> impl Future<Output = bool> + use<> {
let (done, committed) = oneshot::channel();
self.send(Write::SizingResolved {
id,
resolution,
done,
});
async move { committed.await.unwrap_or(false) }
}
pub fn notify_record(
&self,
target: &str,
set: Vec<(String, u64)>,
remove: Vec<String>,
) -> impl Future<Output = bool> + use<> {
let (done, committed) = oneshot::channel();
self.send(Write::Notify {
change: NotifyChange::Record {
target: target.to_owned(),
set,
remove,
},
done,
});
async move { committed.await.unwrap_or(false) }
}
pub fn notify_keep(&self, targets: Vec<String>) -> impl Future<Output = bool> + use<> {
let (done, committed) = oneshot::channel();
self.send(Write::Notify {
change: NotifyChange::Keep(targets),
done,
});
async move { committed.await.unwrap_or(false) }
}
pub fn prune(&self, now: u64, history: &History) -> tokio::task::JoinHandle<()> {
let pruner = self.pruner.clone();
let closed = self.closed.clone();
let history = history.clone();
tokio::task::spawn_blocking(move || {
let db = match pruner.try_lock() {
Ok(db) => db,
Err(TryLockError::Poisoned(poisoned)) => poisoned.into_inner(),
Err(TryLockError::WouldBlock) => return,
};
let db = prune::Db {
connection: &db,
stop: &closed,
};
if !db.stop.load(Ordering::Relaxed) {
prune::run(&db, now, &history);
}
})
}
pub fn close(&self) {
self.closed.store(true, Ordering::Relaxed);
}
pub async fn flush(&self) {
let (done, wait) = oneshot::channel();
self.send(Write::Flush(done));
let _ = wait.await;
}
pub fn subscribe(&self) -> broadcast::Receiver<Arc<EventEnvelope>> {
self.events.subscribe()
}
async fn read<T: Send + 'static>(
&self,
query: impl FnOnce(&Connection) -> rusqlite::Result<T> + Send + 'static,
) -> Result<T, StoreError> {
let reader = self.reader.clone();
Ok(tokio::task::spawn_blocking(move || query(&lock(&reader))).await??)
}
pub async fn jobs(&self, query: api::JobQuery) -> Result<api::JobList, StoreError> {
self.read(move |db| read::jobs(db, &query)).await
}
pub async fn timeline(&self, window: api::Span) -> Result<api::SlotTimeline, StoreError> {
let held: Vec<u64> = lock(&self.held).iter().map(|sample| sample.at).collect();
self.read(move |db| lens::timeline(db, window, &held, api::SAMPLE_INTERVAL))
.await
}
pub async fn compare(
&self,
id: u64,
with: Option<u64>,
) -> Result<Option<api::JobComparison>, StoreError> {
self.read(move |db| lens::compare(db, id, with)).await
}
pub async fn job(&self, id: u64) -> Result<Option<api::Job>, StoreError> {
self.read(move |db| read::job(db, id)).await
}
pub async fn run_jobs(
&self,
repository: &str,
run: u64,
except: u64,
limit: usize,
) -> Result<RunJobs, StoreError> {
let repository = repository.to_owned();
self.read(move |db| run::run_jobs(db, &repository, run, except, limit))
.await
}
pub async fn job_samples(
&self,
id: u64,
interval: Duration,
) -> Result<Option<api::SampleSeries>, StoreError> {
self.read(move |db| read::job_samples(db, id, interval))
.await
}
pub async fn log_lines(
&self,
job: u64,
from: u64,
limit: u64,
) -> Result<Vec<(u64, String)>, StoreError> {
self.read(move |db| read::log_lines(db, job, from, limit))
.await
}
pub async fn log_outline(&self, job: u64) -> Result<Option<api::LogOutline>, StoreError> {
self.read(move |db| outline::outline(db, job)).await
}
pub async fn styled_lines(
&self,
job: u64,
from: u64,
limit: u64,
) -> Result<Vec<api::StyledLine>, StoreError> {
self.read(move |db| outline::styled_lines(db, job, from, limit))
.await
}
pub async fn search_log(
&self,
job: u64,
mut search: api::Search,
) -> Result<api::LogSearch, StoreError> {
while let Some(want) = search.wants() {
search = self
.read(move |db| {
search.read(read::lines_from(db, job, want)?);
Ok(search)
})
.await?;
}
Ok(search.finish())
}
pub async fn events_after(
&self,
after: u64,
limit: u64,
) -> Result<Vec<EventEnvelope>, StoreError> {
self.read(move |db| read::events_after(db, after, limit))
.await
}
pub async fn sizing_changes(&self, since: u64) -> Result<Vec<api::SizingChange>, StoreError> {
self.read(move |db| read::sizing_changes(db, since)).await
}
pub async fn auto_sizing(&self) -> Result<AutoSizing, StoreError> {
self.read(read::auto_sizing).await
}
pub fn announced(&self) -> Result<Announced, StoreError> {
Ok(read::announced(&lock(&self.reader))?)
}
pub fn paused_repositories(&self) -> Result<Vec<String>, StoreError> {
Ok(read::paused(&lock(&self.reader))?)
}
pub async fn host_samples(&self, since: u64) -> Result<Vec<api::HostSample>, StoreError> {
let held = lock(&self.held).clone();
let mut samples = self.read(move |db| read::host_samples(db, since)).await?;
let stored = samples.last().map_or(0, |sample| sample.at);
samples.extend(
held.into_iter()
.filter(|sample| sample.at >= since && sample.at > stored),
);
Ok(samples)
}
pub async fn activity(&self, since: u64, now: u64) -> Result<api::Activity, StoreError> {
self.read(move |db| read::activity(db, since, now)).await
}
pub async fn stats(&self, query: api::StatsQuery, now: u64) -> Result<api::Stats, StoreError> {
self.read(move |db| read::stats(db, &query, now)).await
}
pub async fn sizing(&self, since: u64) -> Result<sizing::History, StoreError> {
self.read(move |db| read::sizing(db, since)).await
}
pub async fn pricing(&self, since: u64) -> Result<api::PricingHistory, StoreError> {
self.read(move |db| read::pricing(db, since)).await
}
pub async fn insights(&self, since: u64) -> Result<crate::insights::History, StoreError> {
let memory = self.latest_host_memory();
let (waiting, seen) = lock(&self.host_waiting).clone();
self.read(move |db| {
let mut history = read::insights(db, since)?;
if let Some(memory) = memory.filter(|_| !waiting.is_empty()) {
let jobs = read::held_jobs(db, &waiting)?;
history.host =
(!jobs.is_empty()).then_some(crate::insights::HostHold { memory, seen, jobs });
}
Ok(history)
})
.await
}
pub async fn last_event(&self) -> Result<u64, StoreError> {
self.read(read::last_event).await
}
pub async fn notify_state(
&self,
) -> Result<BTreeMap<String, BTreeMap<String, u64>>, StoreError> {
self.read(read::notify_state).await
}
pub async fn warm(&self, since: u64) -> Result<Vec<warm::Seen>, StoreError> {
self.read(move |db| read::warm(db, since)).await
}
pub async fn cache_sizes_now(&self) -> Result<Measured, StoreError> {
self.read(read::latest_cache_sizes).await
}
pub async fn build_cache_since(&self, since: u64) -> Result<api::BuildCacheCounts, StoreError> {
self.read(move |db| read::build_cache_since(db, since))
.await
}
pub async fn cache_history(&self, since: u64) -> Result<Vec<api::CacheSize>, StoreError> {
self.read(move |db| read::cache_history(db, since)).await
}
pub async fn open_jobs(&self) -> Result<Vec<OpenJob>, StoreError> {
self.read(read::open_jobs).await
}
pub async fn slot_jobs(&self) -> Result<Vec<SlotJob>, StoreError> {
self.read(read::slot_jobs).await
}
pub async fn last_runner_second(&self, prefix: String) -> Result<Option<u64>, StoreError> {
self.read(move |db| read::last_runner_second(db, &prefix))
.await
}
pub async fn runner_job(&self, runner: &str) -> Result<Option<u64>, StoreError> {
let runner = runner.to_owned();
self.read(move |db| read::runner_job(db, &runner)).await
}
pub async fn runner_admitted(
&self,
runner: &str,
) -> Result<Option<api::Admission>, StoreError> {
let runner = runner.to_owned();
self.read(move |db| read::runner_admission(db, &runner))
.await
}
pub async fn runner_cancel_deadline(&self, runner: &str) -> Result<Option<u64>, StoreError> {
let runner = runner.to_owned();
self.read(move |db| read::runner_cancel_deadline(db, &runner))
.await
}
pub async fn runner_memory_cap(&self, runner: &str) -> Result<Option<u64>, StoreError> {
let runner = runner.to_owned();
self.read(move |db| read::runner_memory_cap(db, &runner))
.await
}
pub async fn log_position(&self, job: u64) -> Result<LogPosition, StoreError> {
self.read(move |db| read::log_position(db, job)).await
}
pub async fn mirror_log_until(&self) -> Result<MirrorPosition, StoreError> {
self.read(read::mirror_log_until).await
}
}
pub fn sizing_at(path: &Path, since: u64) -> Result<Option<sizing::History>, StoreError> {
read_only(path, |db| sizing_of(db, since))
}
pub fn trace_at(
path: &Path,
since: u64,
named_by: u64,
overhead: Option<u64>,
) -> Result<Option<Trace>, StoreError> {
read_only(path, |db| {
if version(db)? == 0 {
return Ok(Trace::default());
}
Ok(read::trace(db, since, named_by, overhead)?)
})
}
pub fn last_queued_at(path: &Path) -> Result<Option<u64>, StoreError> {
let last = read_only(path, |db| {
if version(db)? == 0 {
return Ok(None);
}
Ok(db.query_row("SELECT MAX(queued_at) FROM jobs", [], |row| row.get(0))?)
})?;
Ok(last.flatten())
}
fn read_only<T>(
path: &Path,
read: impl Fn(&Connection) -> Result<T, StoreError>,
) -> Result<Option<T>, StoreError> {
if !path.exists() {
return Ok(None);
}
let read = |path: &str, flags| -> Result<T, StoreError> {
let db = Connection::open_with_flags(path, flags)?;
db.busy_timeout(Duration::from_secs(5))?;
read(&db)
};
let name = path.to_string_lossy();
match read(&name, OpenFlags::SQLITE_OPEN_READ_ONLY) {
Err(StoreError::Sqlite(error))
if matches!(
error.sqlite_error_code(),
Some(rusqlite::ErrorCode::ReadOnly | rusqlite::ErrorCode::CannotOpen)
) =>
{
let mut wal = path.as_os_str().to_owned();
wal.push("-wal");
let wal = PathBuf::from(wal);
if std::fs::metadata(&wal).is_ok_and(|wal| wal.len() > 0) {
return Err(StoreError::Unread(wal));
}
let uri = format!(
"file:{}?mode=ro&immutable=1",
name.replace('%', "%25")
.replace('?', "%3f")
.replace('#', "%23")
);
read(
&uri,
OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_URI,
)
}
history => history,
}
.map(Some)
}
fn sizing_of(db: &Connection, since: u64) -> Result<sizing::History, StoreError> {
match version(db)? {
0 => return Ok(sizing::History::default()),
VERSION => {}
_ => read_as_current(db)?,
}
let mut history = read::sizing(db, since)?;
let auto = read::auto_sizing(db)?;
history.memory_set = auto.memory;
history.slots_set = auto.slots;
history.changes = read::sizing_changes(db, since.saturating_sub(sizing::memory::HISTORY))?;
Ok(history)
}
fn version(db: &Connection) -> Result<u32, StoreError> {
let found: u32 = db.pragma_query_value(None, "user_version", |row| row.get(0))?;
match found {
0 | OLDEST..=VERSION => Ok(found),
found if found > VERSION => Err(StoreError::Newer {
found,
supported: VERSION,
}),
found => Err(StoreError::Older {
found,
oldest: OLDEST,
}),
}
}
fn read_as_current(db: &Connection) -> Result<(), StoreError> {
let current = Connection::open_in_memory()?;
current.execute_batch(SCHEMA)?;
let tables: Vec<(String, String)> = current
.prepare(
"SELECT name, sql FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'",
)?
.query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?
.collect::<rusqlite::Result<_>>()?;
let columns =
|db: &Connection, table: &str| -> rusqlite::Result<Vec<(String, Option<String>)>> {
db.prepare("SELECT name, dflt_value FROM pragma_table_info(?1, 'main')")?
.query_map([table], |row| Ok((row.get(0)?, row.get(1)?)))?
.collect()
};
for (table, sql) in tables {
let found: Vec<String> = columns(db, &table)?
.into_iter()
.map(|(name, _)| name)
.collect();
if found.is_empty() {
db.execute_batch(&sql.replacen("CREATE TABLE", "CREATE TEMP TABLE", 1))?;
continue;
}
let added: Vec<String> = columns(¤t, &table)?
.into_iter()
.filter(|(name, _)| !found.contains(name))
.map(|(name, default)| format!("{} AS {name}", default.as_deref().unwrap_or("NULL")))
.collect();
if !added.is_empty() {
db.execute_batch(&format!(
"CREATE TEMP VIEW {table} AS SELECT *, {} FROM main.{table}",
added.join(", ")
))?;
}
}
Ok(())
}
fn configure(db: &Connection) -> rusqlite::Result<()> {
db.pragma_update(None, "journal_mode", "wal")?;
db.pragma_update(None, "synchronous", "normal")?;
db.pragma_update(None, "foreign_keys", "on")?;
db.busy_timeout(Duration::from_secs(5))
}
fn prepare(db: &mut Connection) -> Result<(), StoreError> {
let found = version(db)?;
if found == 0 {
let transaction = db.transaction()?;
transaction.execute_batch(SCHEMA)?;
transaction.pragma_update(None, "user_version", VERSION)?;
transaction.commit()?;
return Ok(());
}
for (version, migration) in MIGRATIONS.iter().filter(|(version, _)| *version > found) {
let transaction = db.transaction()?;
transaction.execute_batch(migration)?;
transaction.pragma_update(None, "user_version", version)?;
transaction.commit()?;
}
Ok(())
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct Downtime {
from: u64,
until: u64,
}
impl Downtime {
fn before(db: &mut Connection, now: u64) -> Result<Option<Downtime>, StoreError> {
let transaction = db.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?;
let last: Option<u64> = transaction.query_row(
"SELECT max(at) FROM (SELECT max(at) AS at FROM host_samples UNION ALL SELECT max(at) FROM events \
UNION ALL SELECT value FROM meta WHERE key = 'opened_at')",
[],
|row| row.get(0),
)?;
let downtime = last
.filter(|last| *last < now)
.map(|from| Downtime { from, until: now });
if let Some(downtime) = downtime {
let queued: Vec<(u64, u64)> = transaction
.prepare("SELECT id, queued_at FROM jobs WHERE state = 'queued'")?
.query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?
.collect::<rusqlite::Result<_>>()?;
for (job, queued_at) in queued {
transaction.execute(
"UPDATE jobs SET unseen = unseen + ?2 WHERE id = ?1",
rusqlite::params![job, downtime.of(queued_at)],
)?;
}
}
transaction.execute(
"INSERT OR REPLACE INTO meta (key, value) VALUES ('opened_at', ?1)",
[now],
)?;
transaction.commit()?;
Ok(downtime)
}
fn of(self, queued_at: u64) -> u64 {
self.until.saturating_sub(queued_at.max(self.from))
}
fn after(self, queued_at: u64, started_at: u64) -> u64 {
self.of(queued_at)
.saturating_sub(started_at.saturating_sub(queued_at.max(self.from)))
}
}
fn logged<T>(what: &str, result: rusqlite::Result<T>) -> Option<T> {
result
.map_err(|error| warn!("job history: cannot {what}: {error}"))
.ok()
}
#[cfg(test)]
mod tests;