1use std::collections::{BTreeMap, BTreeSet};
8use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt};
9use std::path::{Path, PathBuf};
10use std::sync::atomic::{AtomicBool, Ordering};
11use std::sync::{Arc, Mutex, TryLockError};
12use std::time::Duration;
13
14use rusqlite::{Connection, OpenFlags};
15use tokio::sync::{OwnedSemaphorePermit, Semaphore, broadcast, mpsc, oneshot};
16use tracing::warn;
17
18use crate::admission::trace::Trace;
19use crate::api::{self, Event, EventEnvelope};
20use crate::config::History;
21use crate::github::{JobDetails, QueuedJob};
22use crate::observe::metrics::{CpuHeld, Events};
23use crate::scheduler::warm;
24use crate::sizing;
25use crate::sync::lock;
26
27mod prune;
28mod read;
29mod write;
30
31const SCHEMA: &str = include_str!("store/schema.sql");
33
34const OLDEST: u32 = 15;
36
37const MIGRATIONS: &[(u32, &str)] = &[
40 (
41 16,
42 "ALTER TABLE jobs ADD COLUMN unseen INTEGER NOT NULL DEFAULT 0;",
43 ),
44 (
45 17,
46 "ALTER TABLE jobs ADD COLUMN cpu_periods INTEGER;\n\
47 ALTER TABLE jobs ADD COLUMN cpu_throttled INTEGER;\n\
48 ALTER TABLE jobs ADD COLUMN cpu_stalled INTEGER;\n\
49 ALTER TABLE samples ADD COLUMN cpu_periods INTEGER;\n\
50 ALTER TABLE samples ADD COLUMN cpu_throttled INTEGER;\n\
51 ALTER TABLE samples ADD COLUMN cpu_stalled INTEGER;",
52 ),
53 (
54 18,
55 "ALTER TABLE jobs ADD COLUMN slot_waited INTEGER NOT NULL DEFAULT 0;",
56 ),
57 (
58 19,
59 "ALTER TABLE jobs ADD COLUMN seen_at INTEGER;\n\
60 ALTER TABLE runners ADD COLUMN purpose TEXT;\n\
61 ALTER TABLE runners ADD COLUMN stopped_for TEXT;",
62 ),
63];
64
65const VERSION: u32 = match MIGRATIONS.last() {
67 Some((version, _)) => *version,
68 None => OLDEST,
69};
70
71pub(crate) const LOG_BACKLOG: u64 = 8 << 20;
74
75const HOST_SAMPLE_BATCH: usize = 30;
78
79const EVENT_BUFFER: usize = 4096;
81
82#[derive(Debug, thiserror::Error)]
84pub enum StoreError {
85 #[error("{0}")]
87 Sqlite(#[from] rusqlite::Error),
88 #[error(
90 "the job history database is version {found}, newer than the {supported} this runnerd reads; \
91 run a newer runnerd or move the database aside"
92 )]
93 Newer {
94 found: u32,
96 supported: u32,
98 },
99 #[error(
101 "the job history database is version {found}, from a runnerd before 0.0.19; this runnerd reads versions \
102 from {oldest} on: move the database aside"
103 )]
104 Older {
105 found: u32,
107 oldest: u32,
109 },
110 #[error("{0}")]
112 Io(#[from] std::io::Error),
113 #[error("a database read stopped: {0}")]
115 Join(#[from] tokio::task::JoinError),
116 #[error(
119 "{} holds changes not yet in the job history, which this user cannot read; run runnerd doctor as the user \
120 the daemon runs as",
121 .0.display()
122 )]
123 Unread(PathBuf),
124}
125
126#[derive(Debug, Clone, PartialEq, Eq)]
128pub struct RunnerRecord {
129 pub name: String,
131 pub repository: String,
133 pub profile: String,
135 pub slot: usize,
137 pub container: String,
139 pub started_at: u64,
141 pub memory_cap: Option<u64>,
143 pub cpu_cap: Option<u64>,
145 pub admission: Option<api::Admission>,
147 pub purpose: Option<&'static str>,
149}
150
151#[derive(Debug, Clone, PartialEq, Eq)]
153pub struct SampleRecord {
154 pub runner: String,
156 pub at: u64,
158 pub cpu: u64,
160 pub cpu_usage: u64,
162 pub memory: u64,
164 pub network_received: u64,
166 pub network_sent: u64,
168 pub events: Events,
170 pub cpu_held: CpuHeld,
172}
173
174#[derive(Debug, Clone, Default, PartialEq, Eq)]
176pub struct Measured {
177 pub at: Option<u64>,
179 pub bytes: BTreeMap<String, u64>,
181}
182
183#[derive(Debug, Clone, Default, PartialEq, Eq)]
186pub struct Announced {
187 pub profiles: Vec<api::ProfileStatus>,
189 pub repositories: Vec<api::RepositoryStatus>,
191}
192
193#[derive(Debug, Clone, PartialEq, Eq)]
195pub struct OpenJob {
196 pub id: u64,
198 pub repository: String,
200 pub state: api::JobState,
202 pub runner: Option<String>,
204 pub runner_removed_at: Option<u64>,
206 pub run_id: u64,
208 pub run_attempt: u32,
210 pub workflow: Option<String>,
212 pub queued_at: u64,
214 pub labels: Vec<String>,
216}
217
218#[derive(Debug, Clone, PartialEq, Eq)]
220pub struct SlotJob {
221 pub profile: String,
223 pub slot: usize,
225 pub key: crate::admission::Key,
227}
228
229#[derive(Debug, Clone, Copy, PartialEq, Eq)]
231pub struct LogPosition {
232 pub page: u64,
234 pub offset: u64,
236}
237
238#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
242pub struct MirrorPosition {
243 pub at: u64,
245 pub seen: u64,
247}
248
249#[derive(Debug)]
251enum Write {
252 Queued {
253 repository: String,
254 job: QueuedJob,
255 seen_at: u64,
256 },
257 RunnerStarted(RunnerRecord),
258 RunnerAdmission {
259 runner: String,
260 admission: api::Admission,
261 },
262 RunnerOomKilled(String),
263 RunnerStoppedFor {
264 runner: String,
265 reason: &'static str,
266 },
267 HostWaited {
268 jobs: Vec<u64>,
269 ms: u64,
270 },
271 SlotWaited {
272 jobs: Vec<u64>,
273 ms: u64,
274 },
275 BuildCache {
276 runner: String,
277 counts: api::BuildCacheCounts,
278 },
279 RunnerRemoved {
280 runner: String,
281 at: u64,
282 },
283 RunnersGone {
284 running: Vec<String>,
285 at: u64,
286 },
287 Claimed {
288 job: u64,
289 run_id: u64,
290 runner: String,
291 at: u64,
292 },
293 Details {
294 details: JobDetails,
295 },
296 Elsewhere {
297 details: JobDetails,
298 },
299 Lost {
300 job: u64,
301 at: u64,
302 },
303 Log {
304 job: u64,
305 lines: Vec<String>,
306 position: Option<LogPosition>,
308 max_bytes: u64,
309 _backlog: OwnedSemaphorePermit,
311 },
312 Samples(Vec<SampleRecord>),
313 HostSample(api::HostSample),
314 MirrorBytes {
315 bytes: Vec<(String, u64)>,
316 until: MirrorPosition,
317 done: oneshot::Sender<bool>,
319 },
320 CacheSizes {
321 at: u64,
322 sizes: BTreeMap<String, u64>,
323 },
324 Event(Box<Event>),
326 RecordMade(api::Record),
327 RecordSkipped {
328 job: u64,
329 why: String,
330 },
331 RecordRemoved {
332 job: u64,
333 at: u64,
334 by: api::RecordRemoval,
335 },
336 Paused {
337 repositories: Vec<String>,
338 done: oneshot::Sender<bool>,
340 },
341 Visibility {
342 repository: String,
343 visibility: api::Visibility,
344 at: u64,
345 },
346 SizingIntent {
347 change: api::SizingChange,
348 done: oneshot::Sender<Option<i64>>,
350 },
351 SizingResolved {
352 id: i64,
353 resolution: Resolution,
354 done: oneshot::Sender<bool>,
356 },
357 Notify {
358 change: NotifyChange,
359 done: oneshot::Sender<bool>,
361 },
362 Flush(oneshot::Sender<()>),
363}
364
365#[derive(Debug)]
367enum NotifyChange {
368 Record {
370 target: String,
371 set: Vec<(String, u64)>,
372 remove: Vec<String>,
373 },
374 Keep(Vec<String>),
376}
377
378#[derive(Debug, Clone, Copy, PartialEq, Eq)]
380pub enum Resolution {
381 Done,
383 Failed,
385 Uncertain,
388}
389
390#[derive(Debug, Clone, Default, PartialEq, Eq)]
392pub struct AutoSizing {
393 pub last: BTreeMap<String, u64>,
395 pub pending: Vec<(i64, api::SizingChange)>,
397 pub cpus: Vec<sizing::CpusSet>,
400 pub memory: Vec<sizing::MemorySet>,
402 pub slots: Vec<sizing::SlotsSet>,
405}
406
407#[derive(Debug, Clone)]
409pub struct Store {
410 writes: mpsc::UnboundedSender<Write>,
411 reader: Arc<Mutex<Connection>>,
412 pruner: Arc<Mutex<Connection>>,
414 closed: Arc<AtomicBool>,
416 held: Arc<Mutex<Vec<api::HostSample>>>,
419 latest_host: Arc<Mutex<Option<api::HostSample>>>,
421 latest_host_memory: Arc<Mutex<Option<api::HostMemory>>>,
423 host_waiting: Arc<Mutex<(BTreeSet<u64>, u64)>>,
425 latest_samples: Arc<Mutex<BTreeMap<String, SampleRecord>>>,
427 gone: Arc<Mutex<BTreeSet<String>>>,
429 events: broadcast::Sender<Arc<EventEnvelope>>,
430 backlog: Arc<Semaphore>,
431}
432
433impl Store {
434 pub fn open(path: &Path) -> Result<Store, StoreError> {
439 Store::open_at(path, crate::time::now_ms())
440 }
441
442 fn open_at(path: &Path, now: u64) -> Result<Store, StoreError> {
444 if let Some(dir) = path.parent().filter(|dir| !dir.as_os_str().is_empty()) {
445 std::fs::DirBuilder::new()
446 .recursive(true)
447 .mode(0o700)
448 .create(dir)?;
449 }
450 std::fs::OpenOptions::new()
452 .create(true)
453 .append(true)
454 .mode(0o600)
455 .open(path)?;
456 let mut db = Connection::open(path)?;
457 configure(&db)?;
458 prepare(&mut db)?;
459 let downtime = Downtime::before(&mut db, now)?;
460 let reader = Connection::open_with_flags(
461 path,
462 OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX,
463 )?;
464 configure(&reader)?;
465 let pruner = Connection::open(path)?;
466 configure(&pruner)?;
467 let (events, _) = broadcast::channel(EVENT_BUFFER);
468 let (writes, receiver) = mpsc::unbounded_channel();
469 let thread_events = events.clone();
470 let held = Arc::new(Mutex::new(Vec::new()));
471 let thread_held = held.clone();
472 std::thread::Builder::new()
473 .name("runnerd-store".to_owned())
474 .spawn(move || write::run(db, downtime, receiver, &thread_events, &thread_held))?;
475 Ok(Store {
476 writes,
477 reader: Arc::new(Mutex::new(reader)),
478 pruner: Arc::new(Mutex::new(pruner)),
479 closed: Arc::new(AtomicBool::new(false)),
480 held,
481 latest_host: Arc::new(Mutex::new(None)),
482 latest_host_memory: Arc::new(Mutex::new(None)),
483 host_waiting: Arc::new(Mutex::new((BTreeSet::new(), 0))),
484 latest_samples: Arc::new(Mutex::new(BTreeMap::new())),
485 gone: Arc::new(Mutex::new(BTreeSet::new())),
486 events,
487 backlog: Arc::new(Semaphore::new(
488 usize::try_from(LOG_BACKLOG).unwrap_or(usize::MAX),
489 )),
490 })
491 }
492
493 fn send(&self, write: Write) {
494 let _ = self.writes.send(write);
496 }
497
498 pub fn queued(&self, repository: &str, job: &QueuedJob, seen_at: u64) {
502 self.send(Write::Queued {
503 repository: repository.to_owned(),
504 job: job.clone(),
505 seen_at,
506 });
507 }
508
509 pub fn runner_started(&self, runner: RunnerRecord) {
511 self.send(Write::RunnerStarted(runner));
512 }
513
514 pub fn runner_removed(&self, runner: &str, at: u64) {
516 lock(&self.latest_samples).remove(runner);
517 lock(&self.gone).remove(runner);
518 self.send(Write::RunnerRemoved {
519 runner: runner.to_owned(),
520 at,
521 });
522 }
523
524 pub fn runners_gone(&self, running: Vec<String>, at: u64) {
527 lock(&self.latest_samples).retain(|runner, _| running.contains(runner));
528 lock(&self.gone).retain(|runner| running.contains(runner));
529 self.send(Write::RunnersGone { running, at });
530 }
531
532 pub fn claimed(&self, job: u64, run_id: u64, runner: &str, at: u64) {
534 self.send(Write::Claimed {
535 job,
536 run_id,
537 runner: runner.to_owned(),
538 at,
539 });
540 }
541
542 pub fn details(&self, details: JobDetails) {
544 self.send(Write::Details { details });
545 }
546
547 pub fn elsewhere(&self, details: JobDetails) {
550 self.send(Write::Elsewhere { details });
551 }
552
553 pub fn lost(&self, job: u64, at: u64) {
555 self.send(Write::Lost { job, at });
556 }
557
558 pub async fn log(&self, job: u64, lines: Vec<String>, position: LogPosition, max_bytes: u64) {
561 let size = |line: &String| line.len() as u64 + 1;
562 let mut lines = lines.into_iter().map(|mut line| {
563 if size(&line) > LOG_BACKLOG {
565 let mut end = usize::try_from(LOG_BACKLOG / 2).unwrap_or(usize::MAX);
566 while !line.is_char_boundary(end) {
567 end -= 1;
568 }
569 line.truncate(end);
570 }
571 line
572 });
573 let mut next = lines.next();
574 loop {
575 let (mut part, mut bytes) = (Vec::new(), 0);
576 while let Some(line) = next.take_if(|line| bytes + size(line) <= LOG_BACKLOG) {
577 bytes += size(&line);
578 part.push(line);
579 next = lines.next();
580 }
581 let last = next.is_none();
582 let share = u32::try_from(bytes.max(1)).unwrap_or(u32::MAX);
583 let Ok(backlog) = self.backlog.clone().acquire_many_owned(share).await else {
585 return;
586 };
587 self.send(Write::Log {
588 job,
589 lines: part,
590 position: last.then_some(position),
591 max_bytes,
592 _backlog: backlog,
593 });
594 if last {
595 return;
596 }
597 }
598 }
599
600 pub fn samples(&self, samples: Vec<SampleRecord>) {
602 let mut latest = lock(&self.latest_samples);
603 for sample in &samples {
604 latest.insert(sample.runner.clone(), sample.clone());
605 }
606 drop(latest);
607 self.send(Write::Samples(samples));
608 }
609
610 pub fn host_sample(&self, sample: api::HostSample) {
613 *lock(&self.latest_host) = Some(sample.clone());
614 lock(&self.held).push(sample.clone());
615 self.send(Write::HostSample(sample));
616 }
617
618 pub fn latest_sample(&self, runner: &str) -> Option<SampleRecord> {
620 lock(&self.latest_samples).get(runner).cloned()
621 }
622
623 pub fn containers_gone(&self, runners: Vec<String>) {
626 lock(&self.gone).extend(runners);
627 }
628
629 pub fn containers_read(&self, runners: &[String]) {
632 let mut gone = lock(&self.gone);
633 for runner in runners {
634 gone.remove(runner);
635 }
636 }
637
638 pub fn container_gone(&self, runner: &str) -> bool {
640 lock(&self.gone).contains(runner)
641 }
642
643 pub fn runner_admission(&self, runner: &str, admission: api::Admission) {
645 self.send(Write::RunnerAdmission {
646 runner: runner.to_owned(),
647 admission,
648 });
649 }
650
651 pub fn runner_oom_killed(&self, runner: &str) {
654 self.send(Write::RunnerOomKilled(runner.to_owned()));
655 }
656
657 pub fn runner_stopped_for(&self, runner: &str, reason: &'static str) {
659 self.send(Write::RunnerStoppedFor {
660 runner: runner.to_owned(),
661 reason,
662 });
663 }
664
665 pub fn build_cache(&self, runner: &str, counts: api::BuildCacheCounts) {
667 self.send(Write::BuildCache {
668 runner: runner.to_owned(),
669 counts,
670 });
671 }
672
673 pub fn host_memory(&self, memory: api::HostMemory) {
675 *lock(&self.latest_host_memory) = Some(memory);
676 }
677
678 pub fn latest_host_memory(&self) -> Option<api::HostMemory> {
680 lock(&self.latest_host_memory).clone()
681 }
682
683 pub fn host_waiting(&self, jobs: BTreeSet<u64>, at: u64) {
686 *lock(&self.host_waiting) = (jobs, at);
687 }
688
689 pub fn host_waited(&self, jobs: Vec<u64>, ms: u64) {
692 self.send(Write::HostWaited { jobs, ms });
693 }
694
695 pub fn slot_waited(&self, jobs: Vec<u64>, ms: u64) {
698 self.send(Write::SlotWaited { jobs, ms });
699 }
700
701 pub fn latest_host_sample(&self) -> Option<api::HostSample> {
703 lock(&self.latest_host).clone()
704 }
705
706 pub async fn mirror_bytes(&self, bytes: Vec<(String, u64)>, until: MirrorPosition) -> bool {
709 let (done, committed) = oneshot::channel();
710 self.send(Write::MirrorBytes { bytes, until, done });
711 committed.await.unwrap_or(false)
712 }
713
714 pub fn cache_sizes(&self, at: u64, sizes: BTreeMap<String, u64>) {
716 self.send(Write::CacheSizes { at, sizes });
717 }
718
719 pub fn event(&self, event: Event) {
721 self.send(Write::Event(Box::new(event)));
722 }
723
724 pub fn record_made(&self, record: api::Record) {
726 self.send(Write::RecordMade(record));
727 }
728
729 pub fn record_skipped(&self, job: u64, why: String) {
731 self.send(Write::RecordSkipped { job, why });
732 }
733
734 pub fn record_removed(&self, job: u64, at: u64, by: api::RecordRemoval) {
736 self.send(Write::RecordRemoved { job, at, by });
737 }
738
739 pub fn paused(&self, repositories: Vec<String>) -> impl Future<Output = bool> + use<> {
742 let (done, committed) = oneshot::channel();
743 self.send(Write::Paused { repositories, done });
744 async move { committed.await.unwrap_or(false) }
745 }
746
747 pub fn visibility(&self, repository: String, visibility: api::Visibility, at: u64) {
749 self.send(Write::Visibility {
750 repository,
751 visibility,
752 at,
753 });
754 }
755
756 pub fn sizing_intent(
759 &self,
760 change: api::SizingChange,
761 ) -> impl Future<Output = Option<i64>> + use<> {
762 let (done, committed) = oneshot::channel();
763 self.send(Write::SizingIntent { change, done });
764 async move { committed.await.ok().flatten() }
765 }
766
767 pub fn sizing_resolved(
770 &self,
771 id: i64,
772 resolution: Resolution,
773 ) -> impl Future<Output = bool> + use<> {
774 let (done, committed) = oneshot::channel();
775 self.send(Write::SizingResolved {
776 id,
777 resolution,
778 done,
779 });
780 async move { committed.await.unwrap_or(false) }
781 }
782
783 pub fn notify_record(
786 &self,
787 target: &str,
788 set: Vec<(String, u64)>,
789 remove: Vec<String>,
790 ) -> impl Future<Output = bool> + use<> {
791 let (done, committed) = oneshot::channel();
792 self.send(Write::Notify {
793 change: NotifyChange::Record {
794 target: target.to_owned(),
795 set,
796 remove,
797 },
798 done,
799 });
800 async move { committed.await.unwrap_or(false) }
801 }
802
803 pub fn notify_keep(&self, targets: Vec<String>) -> impl Future<Output = bool> + use<> {
805 let (done, committed) = oneshot::channel();
806 self.send(Write::Notify {
807 change: NotifyChange::Keep(targets),
808 done,
809 });
810 async move { committed.await.unwrap_or(false) }
811 }
812
813 pub fn prune(&self, now: u64, history: &History) -> tokio::task::JoinHandle<()> {
817 let pruner = self.pruner.clone();
818 let closed = self.closed.clone();
819 let history = history.clone();
820 tokio::task::spawn_blocking(move || {
821 let db = match pruner.try_lock() {
822 Ok(db) => db,
823 Err(TryLockError::Poisoned(poisoned)) => poisoned.into_inner(),
824 Err(TryLockError::WouldBlock) => return,
825 };
826 let db = prune::Db {
827 connection: &db,
828 stop: &closed,
829 };
830 if !db.stop.load(Ordering::Relaxed) {
831 prune::run(&db, now, &history);
832 }
833 })
834 }
835
836 pub fn close(&self) {
839 self.closed.store(true, Ordering::Relaxed);
840 }
841
842 pub async fn flush(&self) {
844 let (done, wait) = oneshot::channel();
845 self.send(Write::Flush(done));
846 let _ = wait.await;
847 }
848
849 pub fn subscribe(&self) -> broadcast::Receiver<Arc<EventEnvelope>> {
852 self.events.subscribe()
853 }
854
855 async fn read<T: Send + 'static>(
856 &self,
857 query: impl FnOnce(&Connection) -> rusqlite::Result<T> + Send + 'static,
858 ) -> Result<T, StoreError> {
859 let reader = self.reader.clone();
860 Ok(tokio::task::spawn_blocking(move || query(&lock(&reader))).await??)
862 }
863
864 pub async fn jobs(&self, query: api::JobQuery) -> Result<api::JobPage, StoreError> {
866 self.read(move |db| read::jobs(db, &query)).await
867 }
868
869 pub async fn job(&self, id: u64) -> Result<Option<api::Job>, StoreError> {
871 self.read(move |db| read::job(db, id)).await
872 }
873
874 pub async fn job_samples(
876 &self,
877 id: u64,
878 interval: Duration,
879 ) -> Result<Option<api::SampleSeries>, StoreError> {
880 self.read(move |db| read::job_samples(db, id, interval))
881 .await
882 }
883
884 pub async fn log_lines(
886 &self,
887 job: u64,
888 from: u64,
889 limit: u64,
890 ) -> Result<Vec<(u64, String)>, StoreError> {
891 self.read(move |db| read::log_lines(db, job, from, limit))
892 .await
893 }
894
895 pub async fn events_after(
897 &self,
898 after: u64,
899 limit: u64,
900 ) -> Result<Vec<EventEnvelope>, StoreError> {
901 self.read(move |db| read::events_after(db, after, limit))
902 .await
903 }
904
905 pub async fn sizing_changes(&self, since: u64) -> Result<Vec<api::SizingChange>, StoreError> {
907 self.read(move |db| read::sizing_changes(db, since)).await
908 }
909
910 pub async fn auto_sizing(&self) -> Result<AutoSizing, StoreError> {
912 self.read(read::auto_sizing).await
913 }
914
915 pub fn announced(&self) -> Result<Announced, StoreError> {
918 Ok(read::announced(&lock(&self.reader))?)
919 }
920
921 pub fn paused_repositories(&self) -> Result<Vec<String>, StoreError> {
923 Ok(read::paused(&lock(&self.reader))?)
924 }
925
926 pub async fn host_samples(&self, since: u64) -> Result<Vec<api::HostSample>, StoreError> {
928 let held = lock(&self.held).clone();
931 let mut samples = self.read(move |db| read::host_samples(db, since)).await?;
932 let stored = samples.last().map_or(0, |sample| sample.at);
933 samples.extend(
934 held.into_iter()
935 .filter(|sample| sample.at >= since && sample.at > stored),
936 );
937 Ok(samples)
938 }
939
940 pub async fn activity(&self, since: u64, now: u64) -> Result<api::Activity, StoreError> {
942 self.read(move |db| read::activity(db, since, now)).await
943 }
944
945 pub async fn stats(&self, query: api::StatsQuery, now: u64) -> Result<api::Stats, StoreError> {
947 self.read(move |db| read::stats(db, &query, now)).await
948 }
949
950 pub async fn sizing(&self, since: u64) -> Result<sizing::History, StoreError> {
952 self.read(move |db| read::sizing(db, since)).await
953 }
954
955 pub async fn pricing(&self, since: u64) -> Result<api::PricingHistory, StoreError> {
958 self.read(move |db| read::pricing(db, since)).await
959 }
960
961 pub async fn insights(&self, since: u64) -> Result<crate::insights::History, StoreError> {
964 let memory = self.latest_host_memory();
965 let (waiting, seen) = lock(&self.host_waiting).clone();
966 self.read(move |db| {
967 let mut history = read::insights(db, since)?;
968 if let Some(memory) = memory.filter(|_| !waiting.is_empty()) {
969 let jobs = read::held_jobs(db, &waiting)?;
970 history.host =
971 (!jobs.is_empty()).then_some(crate::insights::HostHold { memory, seen, jobs });
972 }
973 Ok(history)
974 })
975 .await
976 }
977
978 pub async fn last_event(&self) -> Result<u64, StoreError> {
980 self.read(read::last_event).await
981 }
982
983 pub async fn notify_state(
985 &self,
986 ) -> Result<BTreeMap<String, BTreeMap<String, u64>>, StoreError> {
987 self.read(read::notify_state).await
988 }
989
990 pub async fn warm(&self, since: u64) -> Result<Vec<warm::Seen>, StoreError> {
992 self.read(move |db| read::warm(db, since)).await
993 }
994
995 pub async fn cache_sizes_now(&self) -> Result<Measured, StoreError> {
997 self.read(read::latest_cache_sizes).await
998 }
999
1000 pub async fn build_cache_since(&self, since: u64) -> Result<api::BuildCacheCounts, StoreError> {
1002 self.read(move |db| read::build_cache_since(db, since))
1003 .await
1004 }
1005
1006 pub async fn cache_history(&self, since: u64) -> Result<Vec<api::CacheSize>, StoreError> {
1008 self.read(move |db| read::cache_history(db, since)).await
1009 }
1010
1011 pub async fn open_jobs(&self) -> Result<Vec<OpenJob>, StoreError> {
1013 self.read(read::open_jobs).await
1014 }
1015
1016 pub async fn slot_jobs(&self) -> Result<Vec<SlotJob>, StoreError> {
1018 self.read(read::slot_jobs).await
1019 }
1020
1021 pub async fn last_runner_second(&self, prefix: String) -> Result<Option<u64>, StoreError> {
1024 self.read(move |db| read::last_runner_second(db, &prefix))
1025 .await
1026 }
1027
1028 pub async fn runner_job(&self, runner: &str) -> Result<Option<u64>, StoreError> {
1030 let runner = runner.to_owned();
1031 self.read(move |db| read::runner_job(db, &runner)).await
1032 }
1033
1034 pub async fn runner_admitted(
1036 &self,
1037 runner: &str,
1038 ) -> Result<Option<api::Admission>, StoreError> {
1039 let runner = runner.to_owned();
1040 self.read(move |db| read::runner_admission(db, &runner))
1041 .await
1042 }
1043
1044 pub async fn runner_memory_cap(&self, runner: &str) -> Result<Option<u64>, StoreError> {
1046 let runner = runner.to_owned();
1047 self.read(move |db| read::runner_memory_cap(db, &runner))
1048 .await
1049 }
1050
1051 pub async fn log_position(&self, job: u64) -> Result<LogPosition, StoreError> {
1053 self.read(move |db| read::log_position(db, job)).await
1054 }
1055
1056 pub async fn mirror_log_until(&self) -> Result<MirrorPosition, StoreError> {
1058 self.read(read::mirror_log_until).await
1059 }
1060}
1061
1062pub fn sizing_at(path: &Path, since: u64) -> Result<Option<sizing::History>, StoreError> {
1067 read_only(path, |db| sizing_of(db, since))
1068}
1069
1070pub fn trace_at(
1075 path: &Path,
1076 since: u64,
1077 named_by: u64,
1078 overhead: Option<u64>,
1079) -> Result<Option<Trace>, StoreError> {
1080 read_only(path, |db| {
1081 if version(db)? == 0 {
1082 return Ok(Trace::default());
1083 }
1084 Ok(read::trace(db, since, named_by, overhead)?)
1085 })
1086}
1087
1088pub fn last_queued_at(path: &Path) -> Result<Option<u64>, StoreError> {
1091 let last = read_only(path, |db| {
1092 if version(db)? == 0 {
1093 return Ok(None);
1094 }
1095 Ok(db.query_row("SELECT MAX(queued_at) FROM jobs", [], |row| row.get(0))?)
1096 })?;
1097 Ok(last.flatten())
1098}
1099
1100fn read_only<T>(
1102 path: &Path,
1103 read: impl Fn(&Connection) -> Result<T, StoreError>,
1104) -> Result<Option<T>, StoreError> {
1105 if !path.exists() {
1106 return Ok(None);
1107 }
1108 let read = |path: &str, flags| -> Result<T, StoreError> {
1109 let db = Connection::open_with_flags(path, flags)?;
1110 db.busy_timeout(Duration::from_secs(5))?;
1111 read(&db)
1112 };
1113 let name = path.to_string_lossy();
1114 match read(&name, OpenFlags::SQLITE_OPEN_READ_ONLY) {
1115 Err(StoreError::Sqlite(error))
1116 if matches!(
1117 error.sqlite_error_code(),
1118 Some(rusqlite::ErrorCode::ReadOnly | rusqlite::ErrorCode::CannotOpen)
1119 ) =>
1120 {
1121 let mut wal = path.as_os_str().to_owned();
1123 wal.push("-wal");
1124 let wal = PathBuf::from(wal);
1125 if std::fs::metadata(&wal).is_ok_and(|wal| wal.len() > 0) {
1126 return Err(StoreError::Unread(wal));
1127 }
1128 let uri = format!(
1129 "file:{}?mode=ro&immutable=1",
1130 name.replace('%', "%25")
1131 .replace('?', "%3f")
1132 .replace('#', "%23")
1133 );
1134 read(
1135 &uri,
1136 OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_URI,
1137 )
1138 }
1139 history => history,
1140 }
1141 .map(Some)
1142}
1143
1144fn sizing_of(db: &Connection, since: u64) -> Result<sizing::History, StoreError> {
1146 match version(db)? {
1147 0 => return Ok(sizing::History::default()),
1148 VERSION => {}
1149 _ => read_as_current(db)?,
1150 }
1151 let mut history = read::sizing(db, since)?;
1152 let auto = read::auto_sizing(db)?;
1153 history.memory_set = auto.memory;
1154 history.slots_set = auto.slots;
1155 history.changes = read::sizing_changes(db, since.saturating_sub(sizing::memory::HISTORY))?;
1156 Ok(history)
1157}
1158
1159fn version(db: &Connection) -> Result<u32, StoreError> {
1161 let found: u32 = db.pragma_query_value(None, "user_version", |row| row.get(0))?;
1162 match found {
1163 0 | OLDEST..=VERSION => Ok(found),
1164 found if found > VERSION => Err(StoreError::Newer {
1165 found,
1166 supported: VERSION,
1167 }),
1168 found => Err(StoreError::Older {
1169 found,
1170 oldest: OLDEST,
1171 }),
1172 }
1173}
1174
1175fn read_as_current(db: &Connection) -> Result<(), StoreError> {
1179 let current = Connection::open_in_memory()?;
1180 current.execute_batch(SCHEMA)?;
1181 let tables: Vec<(String, String)> = current
1182 .prepare(
1183 "SELECT name, sql FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'",
1184 )?
1185 .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?
1186 .collect::<rusqlite::Result<_>>()?;
1187 let columns =
1188 |db: &Connection, table: &str| -> rusqlite::Result<Vec<(String, Option<String>)>> {
1189 db.prepare("SELECT name, dflt_value FROM pragma_table_info(?1, 'main')")?
1190 .query_map([table], |row| Ok((row.get(0)?, row.get(1)?)))?
1191 .collect()
1192 };
1193 for (table, sql) in tables {
1194 let found: Vec<String> = columns(db, &table)?
1195 .into_iter()
1196 .map(|(name, _)| name)
1197 .collect();
1198 if found.is_empty() {
1199 db.execute_batch(&sql.replacen("CREATE TABLE", "CREATE TEMP TABLE", 1))?;
1200 continue;
1201 }
1202 let added: Vec<String> = columns(¤t, &table)?
1203 .into_iter()
1204 .filter(|(name, _)| !found.contains(name))
1205 .map(|(name, default)| format!("{} AS {name}", default.as_deref().unwrap_or("NULL")))
1206 .collect();
1207 if !added.is_empty() {
1208 db.execute_batch(&format!(
1209 "CREATE TEMP VIEW {table} AS SELECT *, {} FROM main.{table}",
1210 added.join(", ")
1211 ))?;
1212 }
1213 }
1214 Ok(())
1215}
1216
1217fn configure(db: &Connection) -> rusqlite::Result<()> {
1219 db.pragma_update(None, "journal_mode", "wal")?;
1220 db.pragma_update(None, "synchronous", "normal")?;
1221 db.pragma_update(None, "foreign_keys", "on")?;
1222 db.busy_timeout(Duration::from_secs(5))
1223}
1224
1225fn prepare(db: &mut Connection) -> Result<(), StoreError> {
1228 let found = version(db)?;
1229 if found == 0 {
1230 let transaction = db.transaction()?;
1231 transaction.execute_batch(SCHEMA)?;
1232 transaction.pragma_update(None, "user_version", VERSION)?;
1233 transaction.commit()?;
1234 return Ok(());
1235 }
1236 for (version, migration) in MIGRATIONS.iter().filter(|(version, _)| *version > found) {
1237 let transaction = db.transaction()?;
1238 transaction.execute_batch(migration)?;
1239 transaction.pragma_update(None, "user_version", version)?;
1240 transaction.commit()?;
1241 }
1242 Ok(())
1243}
1244
1245#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1250struct Downtime {
1251 from: u64,
1252 until: u64,
1253}
1254
1255impl Downtime {
1256 fn before(db: &mut Connection, now: u64) -> Result<Option<Downtime>, StoreError> {
1260 let transaction = db.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?;
1263 let last: Option<u64> = transaction.query_row(
1264 "SELECT max(at) FROM (SELECT max(at) AS at FROM host_samples UNION ALL SELECT max(at) FROM events \
1265 UNION ALL SELECT value FROM meta WHERE key = 'opened_at')",
1266 [],
1267 |row| row.get(0),
1268 )?;
1269 let downtime = last
1270 .filter(|last| *last < now)
1271 .map(|from| Downtime { from, until: now });
1272 if let Some(downtime) = downtime {
1273 let queued: Vec<(u64, u64)> = transaction
1274 .prepare("SELECT id, queued_at FROM jobs WHERE state = 'queued'")?
1275 .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?
1276 .collect::<rusqlite::Result<_>>()?;
1277 for (job, queued_at) in queued {
1278 transaction.execute(
1279 "UPDATE jobs SET unseen = unseen + ?2 WHERE id = ?1",
1280 rusqlite::params![job, downtime.of(queued_at)],
1281 )?;
1282 }
1283 }
1284 transaction.execute(
1285 "INSERT OR REPLACE INTO meta (key, value) VALUES ('opened_at', ?1)",
1286 [now],
1287 )?;
1288 transaction.commit()?;
1289 Ok(downtime)
1290 }
1291
1292 fn of(self, queued_at: u64) -> u64 {
1294 self.until.saturating_sub(queued_at.max(self.from))
1295 }
1296
1297 fn after(self, queued_at: u64, started_at: u64) -> u64 {
1300 self.of(queued_at)
1301 .saturating_sub(started_at.saturating_sub(queued_at.max(self.from)))
1302 }
1303}
1304
1305fn logged<T>(what: &str, result: rusqlite::Result<T>) -> Option<T> {
1307 result
1308 .map_err(|error| warn!("job history: cannot {what}: {error}"))
1309 .ok()
1310}
1311
1312#[cfg(test)]
1313mod tests;