1use std::collections::{BTreeMap, BTreeSet};
8use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt};
9use std::path::{Path, PathBuf};
10use std::sync::atomic::{AtomicBool, Ordering};
11use std::sync::{Arc, Mutex, TryLockError};
12use std::time::Duration;
13
14use rusqlite::{Connection, OpenFlags};
15use tokio::sync::{OwnedSemaphorePermit, Semaphore, broadcast, mpsc, oneshot};
16use tracing::warn;
17
18use crate::admission::trace::Trace;
19use crate::api::{self, Event, EventEnvelope};
20use crate::config::History;
21use crate::github::{JobDetails, QueuedJob};
22use crate::observe::metrics::{CpuHeld, Events};
23use crate::scheduler::warm;
24use crate::sizing;
25use crate::sync::lock;
26
27mod prune;
28mod read;
29mod write;
30
31const SCHEMA: &str = include_str!("store/schema.sql");
33
34const OLDEST: u32 = 15;
36
37const MIGRATIONS: &[(u32, &str)] = &[
40 (
41 16,
42 "ALTER TABLE jobs ADD COLUMN unseen INTEGER NOT NULL DEFAULT 0;",
43 ),
44 (
45 17,
46 "ALTER TABLE jobs ADD COLUMN cpu_periods INTEGER;\n\
47 ALTER TABLE jobs ADD COLUMN cpu_throttled INTEGER;\n\
48 ALTER TABLE jobs ADD COLUMN cpu_stalled INTEGER;\n\
49 ALTER TABLE samples ADD COLUMN cpu_periods INTEGER;\n\
50 ALTER TABLE samples ADD COLUMN cpu_throttled INTEGER;\n\
51 ALTER TABLE samples ADD COLUMN cpu_stalled INTEGER;",
52 ),
53 (
54 18,
55 "ALTER TABLE jobs ADD COLUMN slot_waited INTEGER NOT NULL DEFAULT 0;",
56 ),
57 (
58 19,
59 "ALTER TABLE jobs ADD COLUMN seen_at INTEGER;\n\
60 ALTER TABLE runners ADD COLUMN purpose TEXT;\n\
61 ALTER TABLE runners ADD COLUMN stopped_for TEXT;",
62 ),
63 (
64 20,
65 "ALTER TABLE jobs ADD COLUMN runner_fault INTEGER NOT NULL DEFAULT 0;",
66 ),
67 (21, "ALTER TABLE runners ADD COLUMN cancel_by INTEGER;"),
68];
69
70const VERSION: u32 = match MIGRATIONS.last() {
72 Some((version, _)) => *version,
73 None => OLDEST,
74};
75
76pub(crate) const LOG_BACKLOG: u64 = 8 << 20;
79
80const HOST_SAMPLE_BATCH: usize = 30;
83
84const EVENT_BUFFER: usize = 4096;
86
87#[derive(Debug, thiserror::Error)]
89pub enum StoreError {
90 #[error("{0}")]
92 Sqlite(#[from] rusqlite::Error),
93 #[error(
95 "the job history database is version {found}, newer than the {supported} this runnerd reads; \
96 run a newer runnerd or move the database aside"
97 )]
98 Newer {
99 found: u32,
101 supported: u32,
103 },
104 #[error(
106 "the job history database is version {found}, from a runnerd before 0.0.19; this runnerd reads versions \
107 from {oldest} on: move the database aside"
108 )]
109 Older {
110 found: u32,
112 oldest: u32,
114 },
115 #[error("{0}")]
117 Io(#[from] std::io::Error),
118 #[error("a database read stopped: {0}")]
120 Join(#[from] tokio::task::JoinError),
121 #[error(
124 "{} holds changes not yet in the job history, which this user cannot read; run runnerd doctor as the user \
125 the daemon runs as",
126 .0.display()
127 )]
128 Unread(PathBuf),
129}
130
131impl StoreError {
132 pub fn is_shutdown(&self) -> bool {
135 matches!(self, StoreError::Join(join) if join.is_cancelled())
136 }
137}
138
139#[derive(Debug, Clone, PartialEq, Eq)]
141pub struct RunnerRecord {
142 pub name: String,
144 pub repository: String,
146 pub profile: String,
148 pub slot: usize,
150 pub container: String,
152 pub started_at: u64,
154 pub memory_cap: Option<u64>,
156 pub cpu_cap: Option<u64>,
158 pub admission: Option<api::Admission>,
160 pub purpose: Option<&'static str>,
162}
163
164#[derive(Debug, Clone, PartialEq, Eq)]
166pub struct SampleRecord {
167 pub runner: String,
169 pub at: u64,
171 pub cpu: u64,
173 pub cpu_usage: u64,
175 pub memory: u64,
177 pub network_received: u64,
179 pub network_sent: u64,
181 pub events: Events,
183 pub cpu_held: CpuHeld,
185}
186
187#[derive(Debug, Clone, Default, PartialEq, Eq)]
189pub struct Measured {
190 pub at: Option<u64>,
192 pub bytes: BTreeMap<String, u64>,
194}
195
196#[derive(Debug, Clone, Default, PartialEq, Eq)]
199pub struct Announced {
200 pub profiles: Vec<api::ProfileStatus>,
202 pub repositories: Vec<api::RepositoryStatus>,
204}
205
206#[derive(Debug, Clone, PartialEq, Eq)]
208pub struct OpenJob {
209 pub id: u64,
211 pub repository: String,
213 pub state: api::JobState,
215 pub runner: Option<String>,
217 pub runner_removed_at: Option<u64>,
219 pub run_id: u64,
221 pub run_attempt: u32,
223 pub workflow: Option<String>,
225 pub queued_at: u64,
227 pub labels: Vec<String>,
229}
230
231#[derive(Debug, Clone, PartialEq, Eq)]
233pub struct SlotJob {
234 pub profile: String,
236 pub slot: usize,
238 pub key: crate::admission::Key,
240}
241
242#[derive(Debug, Clone, Copy, PartialEq, Eq)]
244pub struct LogPosition {
245 pub page: u64,
247 pub offset: u64,
249}
250
251#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
255pub struct MirrorPosition {
256 pub at: u64,
258 pub seen: u64,
260}
261
262#[derive(Debug)]
264enum Write {
265 Queued {
266 repository: String,
267 job: QueuedJob,
268 seen_at: u64,
269 },
270 RunnerStarted(RunnerRecord),
271 RunnerAdmission {
272 runner: String,
273 admission: api::Admission,
274 },
275 RunnerOomKilled(String),
276 RunnerStoppedFor {
277 runner: String,
278 reason: &'static str,
279 },
280 RunnerCancelBy {
281 runner: String,
282 at: u64,
283 },
284 HostWaited {
285 jobs: Vec<u64>,
286 ms: u64,
287 },
288 SlotWaited {
289 jobs: Vec<u64>,
290 ms: u64,
291 },
292 BuildCache {
293 runner: String,
294 counts: api::BuildCacheCounts,
295 },
296 RunnerRemoved {
297 runner: String,
298 at: u64,
299 },
300 RunnersGone {
301 running: Vec<String>,
302 at: u64,
303 },
304 Claimed {
305 job: u64,
306 run_id: u64,
307 runner: String,
308 at: u64,
309 },
310 Details {
311 details: JobDetails,
312 },
313 Elsewhere {
314 details: JobDetails,
315 },
316 Lost {
317 job: u64,
318 at: u64,
319 },
320 Unanswered {
321 job: u64,
322 at: u64,
323 },
324 Log {
325 job: u64,
326 lines: Vec<String>,
327 position: Option<LogPosition>,
329 max_bytes: u64,
330 _backlog: OwnedSemaphorePermit,
332 },
333 Samples(Vec<SampleRecord>),
334 HostSample(api::HostSample),
335 MirrorBytes {
336 bytes: Vec<(String, u64)>,
337 until: MirrorPosition,
338 done: oneshot::Sender<bool>,
340 },
341 CacheSizes {
342 at: u64,
343 sizes: BTreeMap<String, u64>,
344 },
345 Event(Box<Event>),
347 RecordMade(api::Record),
348 RecordSkipped {
349 job: u64,
350 why: String,
351 },
352 RecordRemoved {
353 job: u64,
354 at: u64,
355 by: api::RecordRemoval,
356 },
357 Paused {
358 repositories: Vec<String>,
359 done: oneshot::Sender<bool>,
361 },
362 Visibility {
363 repository: String,
364 visibility: api::Visibility,
365 at: u64,
366 },
367 SizingIntent {
368 change: api::SizingChange,
369 done: oneshot::Sender<Option<i64>>,
371 },
372 SizingResolved {
373 id: i64,
374 resolution: Resolution,
375 done: oneshot::Sender<bool>,
377 },
378 Notify {
379 change: NotifyChange,
380 done: oneshot::Sender<bool>,
382 },
383 RunnerNamed {
384 secs: u64,
385 done: oneshot::Sender<bool>,
387 },
388 Flush(oneshot::Sender<()>),
389}
390
391#[derive(Debug)]
393enum NotifyChange {
394 Record {
396 target: String,
397 set: Vec<(String, u64)>,
398 remove: Vec<String>,
399 },
400 Keep(Vec<String>),
402}
403
404#[derive(Debug, Clone, Copy, PartialEq, Eq)]
406pub enum Resolution {
407 Done,
409 Failed,
411 Uncertain,
414}
415
416#[derive(Debug, Clone, Default, PartialEq, Eq)]
418pub struct AutoSizing {
419 pub last: BTreeMap<String, u64>,
421 pub pending: Vec<(i64, api::SizingChange)>,
423 pub cpus: Vec<sizing::CpusSet>,
426 pub memory: Vec<sizing::MemorySet>,
428 pub slots: Vec<sizing::SlotsSet>,
431}
432
433#[derive(Debug, Clone)]
435pub struct Store {
436 writes: mpsc::UnboundedSender<Write>,
437 reader: Arc<Mutex<Connection>>,
438 pruner: Arc<Mutex<Connection>>,
440 closed: Arc<AtomicBool>,
442 held: Arc<Mutex<Vec<api::HostSample>>>,
445 latest_host: Arc<Mutex<Option<api::HostSample>>>,
447 latest_host_memory: Arc<Mutex<Option<api::HostMemory>>>,
449 host_waiting: Arc<Mutex<(BTreeSet<u64>, u64)>>,
451 latest_samples: Arc<Mutex<BTreeMap<String, SampleRecord>>>,
453 gone: Arc<Mutex<BTreeSet<String>>>,
455 events: broadcast::Sender<Arc<EventEnvelope>>,
456 backlog: Arc<Semaphore>,
457}
458
459impl Store {
460 pub fn open(path: &Path) -> Result<Store, StoreError> {
465 Store::open_at(path, crate::time::now_ms())
466 }
467
468 fn open_at(path: &Path, now: u64) -> Result<Store, StoreError> {
470 if let Some(dir) = path.parent().filter(|dir| !dir.as_os_str().is_empty()) {
471 std::fs::DirBuilder::new()
472 .recursive(true)
473 .mode(0o700)
474 .create(dir)?;
475 }
476 std::fs::OpenOptions::new()
478 .create(true)
479 .append(true)
480 .mode(0o600)
481 .open(path)?;
482 let mut db = Connection::open(path)?;
483 configure(&db)?;
484 prepare(&mut db)?;
485 let downtime = Downtime::before(&mut db, now)?;
486 let reader = Connection::open_with_flags(
487 path,
488 OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX,
489 )?;
490 configure(&reader)?;
491 let pruner = Connection::open(path)?;
492 configure(&pruner)?;
493 let (events, _) = broadcast::channel(EVENT_BUFFER);
494 let (writes, receiver) = mpsc::unbounded_channel();
495 let thread_events = events.clone();
496 let held = Arc::new(Mutex::new(Vec::new()));
497 let thread_held = held.clone();
498 std::thread::Builder::new()
499 .name("runnerd-store".to_owned())
500 .spawn(move || write::run(db, downtime, receiver, &thread_events, &thread_held))?;
501 Ok(Store {
502 writes,
503 reader: Arc::new(Mutex::new(reader)),
504 pruner: Arc::new(Mutex::new(pruner)),
505 closed: Arc::new(AtomicBool::new(false)),
506 held,
507 latest_host: Arc::new(Mutex::new(None)),
508 latest_host_memory: Arc::new(Mutex::new(None)),
509 host_waiting: Arc::new(Mutex::new((BTreeSet::new(), 0))),
510 latest_samples: Arc::new(Mutex::new(BTreeMap::new())),
511 gone: Arc::new(Mutex::new(BTreeSet::new())),
512 events,
513 backlog: Arc::new(Semaphore::new(
514 usize::try_from(LOG_BACKLOG).unwrap_or(usize::MAX),
515 )),
516 })
517 }
518
519 fn send(&self, write: Write) {
520 let _ = self.writes.send(write);
522 }
523
524 pub fn queued(&self, repository: &str, job: &QueuedJob, seen_at: u64) {
528 self.send(Write::Queued {
529 repository: repository.to_owned(),
530 job: job.clone(),
531 seen_at,
532 });
533 }
534
535 pub fn runner_started(&self, runner: RunnerRecord) {
537 self.send(Write::RunnerStarted(runner));
538 }
539
540 pub fn runner_removed(&self, runner: &str, at: u64) {
542 lock(&self.latest_samples).remove(runner);
543 lock(&self.gone).remove(runner);
544 self.send(Write::RunnerRemoved {
545 runner: runner.to_owned(),
546 at,
547 });
548 }
549
550 pub fn runners_gone(&self, running: Vec<String>, at: u64) {
553 lock(&self.latest_samples).retain(|runner, _| running.contains(runner));
554 lock(&self.gone).retain(|runner| running.contains(runner));
555 self.send(Write::RunnersGone { running, at });
556 }
557
558 pub fn claimed(&self, job: u64, run_id: u64, runner: &str, at: u64) {
560 self.send(Write::Claimed {
561 job,
562 run_id,
563 runner: runner.to_owned(),
564 at,
565 });
566 }
567
568 pub fn details(&self, details: JobDetails) {
570 self.send(Write::Details { details });
571 }
572
573 pub fn elsewhere(&self, details: JobDetails) {
576 self.send(Write::Elsewhere { details });
577 }
578
579 pub fn unanswered(&self, job: u64, at: u64) {
582 self.send(Write::Unanswered { job, at });
583 }
584
585 pub fn lost(&self, job: u64, at: u64) {
587 self.send(Write::Lost { job, at });
588 }
589
590 pub async fn log(&self, job: u64, lines: Vec<String>, position: LogPosition, max_bytes: u64) {
593 let size = |line: &String| line.len() as u64 + 1;
594 let mut lines = lines.into_iter().map(|mut line| {
595 if size(&line) > LOG_BACKLOG {
597 let mut end = usize::try_from(LOG_BACKLOG / 2).unwrap_or(usize::MAX);
598 while !line.is_char_boundary(end) {
599 end -= 1;
600 }
601 line.truncate(end);
602 }
603 line
604 });
605 let mut next = lines.next();
606 loop {
607 let (mut part, mut bytes) = (Vec::new(), 0);
608 while let Some(line) = next.take_if(|line| bytes + size(line) <= LOG_BACKLOG) {
609 bytes += size(&line);
610 part.push(line);
611 next = lines.next();
612 }
613 let last = next.is_none();
614 let share = u32::try_from(bytes.max(1)).unwrap_or(u32::MAX);
615 let Ok(backlog) = self.backlog.clone().acquire_many_owned(share).await else {
617 return;
618 };
619 self.send(Write::Log {
620 job,
621 lines: part,
622 position: last.then_some(position),
623 max_bytes,
624 _backlog: backlog,
625 });
626 if last {
627 return;
628 }
629 }
630 }
631
632 pub fn samples(&self, samples: Vec<SampleRecord>) {
634 let mut latest = lock(&self.latest_samples);
635 for sample in &samples {
636 latest.insert(sample.runner.clone(), sample.clone());
637 }
638 drop(latest);
639 self.send(Write::Samples(samples));
640 }
641
642 pub fn host_sample(&self, sample: api::HostSample) {
645 *lock(&self.latest_host) = Some(sample.clone());
646 lock(&self.held).push(sample.clone());
647 self.send(Write::HostSample(sample));
648 }
649
650 pub fn latest_sample(&self, runner: &str) -> Option<SampleRecord> {
652 lock(&self.latest_samples).get(runner).cloned()
653 }
654
655 pub fn containers_gone(&self, runners: Vec<String>) {
658 lock(&self.gone).extend(runners);
659 }
660
661 pub fn containers_read(&self, runners: &[String]) {
664 let mut gone = lock(&self.gone);
665 for runner in runners {
666 gone.remove(runner);
667 }
668 }
669
670 pub fn container_gone(&self, runner: &str) -> bool {
672 lock(&self.gone).contains(runner)
673 }
674
675 pub fn runner_admission(&self, runner: &str, admission: api::Admission) {
677 self.send(Write::RunnerAdmission {
678 runner: runner.to_owned(),
679 admission,
680 });
681 }
682
683 pub fn runner_oom_killed(&self, runner: &str) {
686 self.send(Write::RunnerOomKilled(runner.to_owned()));
687 }
688
689 pub fn runner_cancel_by(&self, runner: &str, at: u64) {
692 self.send(Write::RunnerCancelBy {
693 runner: runner.to_owned(),
694 at,
695 });
696 }
697
698 pub fn runner_stopped_for(&self, runner: &str, reason: &'static str) {
700 self.send(Write::RunnerStoppedFor {
701 runner: runner.to_owned(),
702 reason,
703 });
704 }
705
706 pub fn build_cache(&self, runner: &str, counts: api::BuildCacheCounts) {
708 self.send(Write::BuildCache {
709 runner: runner.to_owned(),
710 counts,
711 });
712 }
713
714 pub fn host_memory(&self, memory: api::HostMemory) {
716 *lock(&self.latest_host_memory) = Some(memory);
717 }
718
719 pub fn latest_host_memory(&self) -> Option<api::HostMemory> {
721 lock(&self.latest_host_memory).clone()
722 }
723
724 pub fn host_waiting(&self, jobs: BTreeSet<u64>, at: u64) {
727 *lock(&self.host_waiting) = (jobs, at);
728 }
729
730 pub fn host_waited(&self, jobs: Vec<u64>, ms: u64) {
733 self.send(Write::HostWaited { jobs, ms });
734 }
735
736 pub fn slot_waited(&self, jobs: Vec<u64>, ms: u64) {
739 self.send(Write::SlotWaited { jobs, ms });
740 }
741
742 pub fn latest_host_sample(&self) -> Option<api::HostSample> {
744 lock(&self.latest_host).clone()
745 }
746
747 pub async fn mirror_bytes(&self, bytes: Vec<(String, u64)>, until: MirrorPosition) -> bool {
750 let (done, committed) = oneshot::channel();
751 self.send(Write::MirrorBytes { bytes, until, done });
752 committed.await.unwrap_or(false)
753 }
754
755 pub fn cache_sizes(&self, at: u64, sizes: BTreeMap<String, u64>) {
757 self.send(Write::CacheSizes { at, sizes });
758 }
759
760 pub fn event(&self, event: Event) {
762 self.send(Write::Event(Box::new(event)));
763 }
764
765 pub fn record_made(&self, record: api::Record) {
767 self.send(Write::RecordMade(record));
768 }
769
770 pub fn record_skipped(&self, job: u64, why: String) {
772 self.send(Write::RecordSkipped { job, why });
773 }
774
775 pub fn record_removed(&self, job: u64, at: u64, by: api::RecordRemoval) {
777 self.send(Write::RecordRemoved { job, at, by });
778 }
779
780 pub fn paused(&self, repositories: Vec<String>) -> impl Future<Output = bool> + use<> {
783 let (done, committed) = oneshot::channel();
784 self.send(Write::Paused { repositories, done });
785 async move { committed.await.unwrap_or(false) }
786 }
787
788 pub async fn runner_named(&self, secs: u64) -> bool {
792 let (done, committed) = oneshot::channel();
793 self.send(Write::RunnerNamed { secs, done });
794 tokio::task::spawn_blocking(move || committed.blocking_recv().unwrap_or(false))
797 .await
798 .unwrap_or(false)
799 }
800
801 pub fn visibility(&self, repository: String, visibility: api::Visibility, at: u64) {
803 self.send(Write::Visibility {
804 repository,
805 visibility,
806 at,
807 });
808 }
809
810 pub fn sizing_intent(
813 &self,
814 change: api::SizingChange,
815 ) -> impl Future<Output = Option<i64>> + use<> {
816 let (done, committed) = oneshot::channel();
817 self.send(Write::SizingIntent { change, done });
818 async move { committed.await.ok().flatten() }
819 }
820
821 pub fn sizing_resolved(
824 &self,
825 id: i64,
826 resolution: Resolution,
827 ) -> impl Future<Output = bool> + use<> {
828 let (done, committed) = oneshot::channel();
829 self.send(Write::SizingResolved {
830 id,
831 resolution,
832 done,
833 });
834 async move { committed.await.unwrap_or(false) }
835 }
836
837 pub fn notify_record(
840 &self,
841 target: &str,
842 set: Vec<(String, u64)>,
843 remove: Vec<String>,
844 ) -> impl Future<Output = bool> + use<> {
845 let (done, committed) = oneshot::channel();
846 self.send(Write::Notify {
847 change: NotifyChange::Record {
848 target: target.to_owned(),
849 set,
850 remove,
851 },
852 done,
853 });
854 async move { committed.await.unwrap_or(false) }
855 }
856
857 pub fn notify_keep(&self, targets: Vec<String>) -> impl Future<Output = bool> + use<> {
859 let (done, committed) = oneshot::channel();
860 self.send(Write::Notify {
861 change: NotifyChange::Keep(targets),
862 done,
863 });
864 async move { committed.await.unwrap_or(false) }
865 }
866
867 pub fn prune(&self, now: u64, history: &History) -> tokio::task::JoinHandle<()> {
871 let pruner = self.pruner.clone();
872 let closed = self.closed.clone();
873 let history = history.clone();
874 tokio::task::spawn_blocking(move || {
875 let db = match pruner.try_lock() {
876 Ok(db) => db,
877 Err(TryLockError::Poisoned(poisoned)) => poisoned.into_inner(),
878 Err(TryLockError::WouldBlock) => return,
879 };
880 let db = prune::Db {
881 connection: &db,
882 stop: &closed,
883 };
884 if !db.stop.load(Ordering::Relaxed) {
885 prune::run(&db, now, &history);
886 }
887 })
888 }
889
890 pub fn close(&self) {
893 self.closed.store(true, Ordering::Relaxed);
894 }
895
896 pub async fn flush(&self) {
898 let (done, wait) = oneshot::channel();
899 self.send(Write::Flush(done));
900 let _ = wait.await;
901 }
902
903 pub fn subscribe(&self) -> broadcast::Receiver<Arc<EventEnvelope>> {
906 self.events.subscribe()
907 }
908
909 async fn read<T: Send + 'static>(
910 &self,
911 query: impl FnOnce(&Connection) -> rusqlite::Result<T> + Send + 'static,
912 ) -> Result<T, StoreError> {
913 let reader = self.reader.clone();
914 Ok(tokio::task::spawn_blocking(move || query(&lock(&reader))).await??)
916 }
917
918 pub async fn jobs(&self, query: api::JobQuery) -> Result<api::JobPage, StoreError> {
920 self.read(move |db| read::jobs(db, &query)).await
921 }
922
923 pub async fn job(&self, id: u64) -> Result<Option<api::Job>, StoreError> {
925 self.read(move |db| read::job(db, id)).await
926 }
927
928 pub async fn job_samples(
930 &self,
931 id: u64,
932 interval: Duration,
933 ) -> Result<Option<api::SampleSeries>, StoreError> {
934 self.read(move |db| read::job_samples(db, id, interval))
935 .await
936 }
937
938 pub async fn log_lines(
940 &self,
941 job: u64,
942 from: u64,
943 limit: u64,
944 ) -> Result<Vec<(u64, String)>, StoreError> {
945 self.read(move |db| read::log_lines(db, job, from, limit))
946 .await
947 }
948
949 pub async fn events_after(
951 &self,
952 after: u64,
953 limit: u64,
954 ) -> Result<Vec<EventEnvelope>, StoreError> {
955 self.read(move |db| read::events_after(db, after, limit))
956 .await
957 }
958
959 pub async fn sizing_changes(&self, since: u64) -> Result<Vec<api::SizingChange>, StoreError> {
961 self.read(move |db| read::sizing_changes(db, since)).await
962 }
963
964 pub async fn auto_sizing(&self) -> Result<AutoSizing, StoreError> {
966 self.read(read::auto_sizing).await
967 }
968
969 pub fn announced(&self) -> Result<Announced, StoreError> {
972 Ok(read::announced(&lock(&self.reader))?)
973 }
974
975 pub fn paused_repositories(&self) -> Result<Vec<String>, StoreError> {
977 Ok(read::paused(&lock(&self.reader))?)
978 }
979
980 pub async fn host_samples(&self, since: u64) -> Result<Vec<api::HostSample>, StoreError> {
982 let held = lock(&self.held).clone();
985 let mut samples = self.read(move |db| read::host_samples(db, since)).await?;
986 let stored = samples.last().map_or(0, |sample| sample.at);
987 samples.extend(
988 held.into_iter()
989 .filter(|sample| sample.at >= since && sample.at > stored),
990 );
991 Ok(samples)
992 }
993
994 pub async fn activity(&self, since: u64, now: u64) -> Result<api::Activity, StoreError> {
996 self.read(move |db| read::activity(db, since, now)).await
997 }
998
999 pub async fn stats(&self, query: api::StatsQuery, now: u64) -> Result<api::Stats, StoreError> {
1001 self.read(move |db| read::stats(db, &query, now)).await
1002 }
1003
1004 pub async fn sizing(&self, since: u64) -> Result<sizing::History, StoreError> {
1006 self.read(move |db| read::sizing(db, since)).await
1007 }
1008
1009 pub async fn pricing(&self, since: u64) -> Result<api::PricingHistory, StoreError> {
1012 self.read(move |db| read::pricing(db, since)).await
1013 }
1014
1015 pub async fn insights(&self, since: u64) -> Result<crate::insights::History, StoreError> {
1018 let memory = self.latest_host_memory();
1019 let (waiting, seen) = lock(&self.host_waiting).clone();
1020 self.read(move |db| {
1021 let mut history = read::insights(db, since)?;
1022 if let Some(memory) = memory.filter(|_| !waiting.is_empty()) {
1023 let jobs = read::held_jobs(db, &waiting)?;
1024 history.host =
1025 (!jobs.is_empty()).then_some(crate::insights::HostHold { memory, seen, jobs });
1026 }
1027 Ok(history)
1028 })
1029 .await
1030 }
1031
1032 pub async fn last_event(&self) -> Result<u64, StoreError> {
1034 self.read(read::last_event).await
1035 }
1036
1037 pub async fn notify_state(
1039 &self,
1040 ) -> Result<BTreeMap<String, BTreeMap<String, u64>>, StoreError> {
1041 self.read(read::notify_state).await
1042 }
1043
1044 pub async fn warm(&self, since: u64) -> Result<Vec<warm::Seen>, StoreError> {
1046 self.read(move |db| read::warm(db, since)).await
1047 }
1048
1049 pub async fn cache_sizes_now(&self) -> Result<Measured, StoreError> {
1051 self.read(read::latest_cache_sizes).await
1052 }
1053
1054 pub async fn build_cache_since(&self, since: u64) -> Result<api::BuildCacheCounts, StoreError> {
1056 self.read(move |db| read::build_cache_since(db, since))
1057 .await
1058 }
1059
1060 pub async fn cache_history(&self, since: u64) -> Result<Vec<api::CacheSize>, StoreError> {
1062 self.read(move |db| read::cache_history(db, since)).await
1063 }
1064
1065 pub async fn open_jobs(&self) -> Result<Vec<OpenJob>, StoreError> {
1067 self.read(read::open_jobs).await
1068 }
1069
1070 pub async fn slot_jobs(&self) -> Result<Vec<SlotJob>, StoreError> {
1072 self.read(read::slot_jobs).await
1073 }
1074
1075 pub async fn last_runner_second(&self, prefix: String) -> Result<Option<u64>, StoreError> {
1079 self.read(move |db| read::last_runner_second(db, &prefix))
1080 .await
1081 }
1082
1083 pub async fn runner_job(&self, runner: &str) -> Result<Option<u64>, StoreError> {
1085 let runner = runner.to_owned();
1086 self.read(move |db| read::runner_job(db, &runner)).await
1087 }
1088
1089 pub async fn runner_admitted(
1091 &self,
1092 runner: &str,
1093 ) -> Result<Option<api::Admission>, StoreError> {
1094 let runner = runner.to_owned();
1095 self.read(move |db| read::runner_admission(db, &runner))
1096 .await
1097 }
1098
1099 pub async fn runner_cancel_deadline(&self, runner: &str) -> Result<Option<u64>, StoreError> {
1102 let runner = runner.to_owned();
1103 self.read(move |db| read::runner_cancel_deadline(db, &runner))
1104 .await
1105 }
1106
1107 pub async fn runner_memory_cap(&self, runner: &str) -> Result<Option<u64>, StoreError> {
1109 let runner = runner.to_owned();
1110 self.read(move |db| read::runner_memory_cap(db, &runner))
1111 .await
1112 }
1113
1114 pub async fn log_position(&self, job: u64) -> Result<LogPosition, StoreError> {
1116 self.read(move |db| read::log_position(db, job)).await
1117 }
1118
1119 pub async fn mirror_log_until(&self) -> Result<MirrorPosition, StoreError> {
1121 self.read(read::mirror_log_until).await
1122 }
1123}
1124
1125pub fn sizing_at(path: &Path, since: u64) -> Result<Option<sizing::History>, StoreError> {
1130 read_only(path, |db| sizing_of(db, since))
1131}
1132
1133pub fn trace_at(
1138 path: &Path,
1139 since: u64,
1140 named_by: u64,
1141 overhead: Option<u64>,
1142) -> Result<Option<Trace>, StoreError> {
1143 read_only(path, |db| {
1144 if version(db)? == 0 {
1145 return Ok(Trace::default());
1146 }
1147 Ok(read::trace(db, since, named_by, overhead)?)
1148 })
1149}
1150
1151pub fn last_queued_at(path: &Path) -> Result<Option<u64>, StoreError> {
1154 let last = read_only(path, |db| {
1155 if version(db)? == 0 {
1156 return Ok(None);
1157 }
1158 Ok(db.query_row("SELECT MAX(queued_at) FROM jobs", [], |row| row.get(0))?)
1159 })?;
1160 Ok(last.flatten())
1161}
1162
1163fn read_only<T>(
1165 path: &Path,
1166 read: impl Fn(&Connection) -> Result<T, StoreError>,
1167) -> Result<Option<T>, StoreError> {
1168 if !path.exists() {
1169 return Ok(None);
1170 }
1171 let read = |path: &str, flags| -> Result<T, StoreError> {
1172 let db = Connection::open_with_flags(path, flags)?;
1173 db.busy_timeout(Duration::from_secs(5))?;
1174 read(&db)
1175 };
1176 let name = path.to_string_lossy();
1177 match read(&name, OpenFlags::SQLITE_OPEN_READ_ONLY) {
1178 Err(StoreError::Sqlite(error))
1179 if matches!(
1180 error.sqlite_error_code(),
1181 Some(rusqlite::ErrorCode::ReadOnly | rusqlite::ErrorCode::CannotOpen)
1182 ) =>
1183 {
1184 let mut wal = path.as_os_str().to_owned();
1186 wal.push("-wal");
1187 let wal = PathBuf::from(wal);
1188 if std::fs::metadata(&wal).is_ok_and(|wal| wal.len() > 0) {
1189 return Err(StoreError::Unread(wal));
1190 }
1191 let uri = format!(
1192 "file:{}?mode=ro&immutable=1",
1193 name.replace('%', "%25")
1194 .replace('?', "%3f")
1195 .replace('#', "%23")
1196 );
1197 read(
1198 &uri,
1199 OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_URI,
1200 )
1201 }
1202 history => history,
1203 }
1204 .map(Some)
1205}
1206
1207fn sizing_of(db: &Connection, since: u64) -> Result<sizing::History, StoreError> {
1209 match version(db)? {
1210 0 => return Ok(sizing::History::default()),
1211 VERSION => {}
1212 _ => read_as_current(db)?,
1213 }
1214 let mut history = read::sizing(db, since)?;
1215 let auto = read::auto_sizing(db)?;
1216 history.memory_set = auto.memory;
1217 history.slots_set = auto.slots;
1218 history.changes = read::sizing_changes(db, since.saturating_sub(sizing::memory::HISTORY))?;
1219 Ok(history)
1220}
1221
1222fn version(db: &Connection) -> Result<u32, StoreError> {
1224 let found: u32 = db.pragma_query_value(None, "user_version", |row| row.get(0))?;
1225 match found {
1226 0 | OLDEST..=VERSION => Ok(found),
1227 found if found > VERSION => Err(StoreError::Newer {
1228 found,
1229 supported: VERSION,
1230 }),
1231 found => Err(StoreError::Older {
1232 found,
1233 oldest: OLDEST,
1234 }),
1235 }
1236}
1237
1238fn read_as_current(db: &Connection) -> Result<(), StoreError> {
1242 let current = Connection::open_in_memory()?;
1243 current.execute_batch(SCHEMA)?;
1244 let tables: Vec<(String, String)> = current
1245 .prepare(
1246 "SELECT name, sql FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'",
1247 )?
1248 .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?
1249 .collect::<rusqlite::Result<_>>()?;
1250 let columns =
1251 |db: &Connection, table: &str| -> rusqlite::Result<Vec<(String, Option<String>)>> {
1252 db.prepare("SELECT name, dflt_value FROM pragma_table_info(?1, 'main')")?
1253 .query_map([table], |row| Ok((row.get(0)?, row.get(1)?)))?
1254 .collect()
1255 };
1256 for (table, sql) in tables {
1257 let found: Vec<String> = columns(db, &table)?
1258 .into_iter()
1259 .map(|(name, _)| name)
1260 .collect();
1261 if found.is_empty() {
1262 db.execute_batch(&sql.replacen("CREATE TABLE", "CREATE TEMP TABLE", 1))?;
1263 continue;
1264 }
1265 let added: Vec<String> = columns(¤t, &table)?
1266 .into_iter()
1267 .filter(|(name, _)| !found.contains(name))
1268 .map(|(name, default)| format!("{} AS {name}", default.as_deref().unwrap_or("NULL")))
1269 .collect();
1270 if !added.is_empty() {
1271 db.execute_batch(&format!(
1272 "CREATE TEMP VIEW {table} AS SELECT *, {} FROM main.{table}",
1273 added.join(", ")
1274 ))?;
1275 }
1276 }
1277 Ok(())
1278}
1279
1280fn configure(db: &Connection) -> rusqlite::Result<()> {
1282 db.pragma_update(None, "journal_mode", "wal")?;
1283 db.pragma_update(None, "synchronous", "normal")?;
1284 db.pragma_update(None, "foreign_keys", "on")?;
1285 db.busy_timeout(Duration::from_secs(5))
1286}
1287
1288fn prepare(db: &mut Connection) -> Result<(), StoreError> {
1291 let found = version(db)?;
1292 if found == 0 {
1293 let transaction = db.transaction()?;
1294 transaction.execute_batch(SCHEMA)?;
1295 transaction.pragma_update(None, "user_version", VERSION)?;
1296 transaction.commit()?;
1297 return Ok(());
1298 }
1299 for (version, migration) in MIGRATIONS.iter().filter(|(version, _)| *version > found) {
1300 let transaction = db.transaction()?;
1301 transaction.execute_batch(migration)?;
1302 transaction.pragma_update(None, "user_version", version)?;
1303 transaction.commit()?;
1304 }
1305 Ok(())
1306}
1307
1308#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1313struct Downtime {
1314 from: u64,
1315 until: u64,
1316}
1317
1318impl Downtime {
1319 fn before(db: &mut Connection, now: u64) -> Result<Option<Downtime>, StoreError> {
1323 let transaction = db.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?;
1326 let last: Option<u64> = transaction.query_row(
1327 "SELECT max(at) FROM (SELECT max(at) AS at FROM host_samples UNION ALL SELECT max(at) FROM events \
1328 UNION ALL SELECT value FROM meta WHERE key = 'opened_at')",
1329 [],
1330 |row| row.get(0),
1331 )?;
1332 let downtime = last
1333 .filter(|last| *last < now)
1334 .map(|from| Downtime { from, until: now });
1335 if let Some(downtime) = downtime {
1336 let queued: Vec<(u64, u64)> = transaction
1337 .prepare("SELECT id, queued_at FROM jobs WHERE state = 'queued'")?
1338 .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?
1339 .collect::<rusqlite::Result<_>>()?;
1340 for (job, queued_at) in queued {
1341 transaction.execute(
1342 "UPDATE jobs SET unseen = unseen + ?2 WHERE id = ?1",
1343 rusqlite::params![job, downtime.of(queued_at)],
1344 )?;
1345 }
1346 }
1347 transaction.execute(
1348 "INSERT OR REPLACE INTO meta (key, value) VALUES ('opened_at', ?1)",
1349 [now],
1350 )?;
1351 transaction.commit()?;
1352 Ok(downtime)
1353 }
1354
1355 fn of(self, queued_at: u64) -> u64 {
1357 self.until.saturating_sub(queued_at.max(self.from))
1358 }
1359
1360 fn after(self, queued_at: u64, started_at: u64) -> u64 {
1363 self.of(queued_at)
1364 .saturating_sub(started_at.saturating_sub(queued_at.max(self.from)))
1365 }
1366}
1367
1368fn logged<T>(what: &str, result: rusqlite::Result<T>) -> Option<T> {
1370 result
1371 .map_err(|error| warn!("job history: cannot {what}: {error}"))
1372 .ok()
1373}
1374
1375#[cfg(test)]
1376mod tests;