1use std::collections::BTreeMap;
8use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt};
9use std::path::{Path, PathBuf};
10use std::sync::atomic::{AtomicBool, Ordering};
11use std::sync::{Arc, Mutex, TryLockError};
12use std::time::Duration;
13
14use rusqlite::{Connection, OpenFlags};
15use tokio::sync::{OwnedSemaphorePermit, Semaphore, broadcast, mpsc, oneshot};
16use tracing::warn;
17
18use crate::api::{self, Event, EventEnvelope};
19use crate::config::History;
20use crate::github::{JobDetails, QueuedJob};
21use crate::observe::metrics::{CpuHeld, Events};
22use crate::scheduler::warm;
23use crate::sizing;
24use crate::sync::lock;
25
26mod prune;
27mod read;
28mod write;
29
30const SCHEMA: &str = include_str!("store/schema.sql");
32
33const OLDEST: u32 = 15;
35
36const MIGRATIONS: &[(u32, &str)] = &[
39 (
40 16,
41 "ALTER TABLE jobs ADD COLUMN unseen INTEGER NOT NULL DEFAULT 0;",
42 ),
43 (
44 17,
45 "ALTER TABLE jobs ADD COLUMN cpu_periods INTEGER;\n\
46 ALTER TABLE jobs ADD COLUMN cpu_throttled INTEGER;\n\
47 ALTER TABLE jobs ADD COLUMN cpu_stalled INTEGER;\n\
48 ALTER TABLE samples ADD COLUMN cpu_periods INTEGER;\n\
49 ALTER TABLE samples ADD COLUMN cpu_throttled INTEGER;\n\
50 ALTER TABLE samples ADD COLUMN cpu_stalled INTEGER;",
51 ),
52 (
53 18,
54 "ALTER TABLE jobs ADD COLUMN slot_waited INTEGER NOT NULL DEFAULT 0;",
55 ),
56 (
57 19,
58 "ALTER TABLE jobs ADD COLUMN seen_at INTEGER;\n\
59 ALTER TABLE runners ADD COLUMN purpose TEXT;\n\
60 ALTER TABLE runners ADD COLUMN stopped_for TEXT;",
61 ),
62];
63
64const VERSION: u32 = match MIGRATIONS.last() {
66 Some((version, _)) => *version,
67 None => OLDEST,
68};
69
70pub(crate) const LOG_BACKLOG: u64 = 8 << 20;
73
74const HOST_SAMPLE_BATCH: usize = 30;
77
78const EVENT_BUFFER: usize = 4096;
80
81#[derive(Debug, thiserror::Error)]
83pub enum StoreError {
84 #[error("{0}")]
86 Sqlite(#[from] rusqlite::Error),
87 #[error(
89 "the job history database is version {found}, newer than the {supported} this runnerd reads; \
90 run a newer runnerd or move the database aside"
91 )]
92 Newer {
93 found: u32,
95 supported: u32,
97 },
98 #[error(
100 "the job history database is version {found}, from a runnerd before 0.0.19; this runnerd reads versions \
101 from {oldest} on: move the database aside"
102 )]
103 Older {
104 found: u32,
106 oldest: u32,
108 },
109 #[error("{0}")]
111 Io(#[from] std::io::Error),
112 #[error("a database read stopped: {0}")]
114 Join(#[from] tokio::task::JoinError),
115 #[error(
118 "{} holds changes not yet in the job history, which this user cannot read; run runnerd doctor as the user \
119 the daemon runs as",
120 .0.display()
121 )]
122 Unread(PathBuf),
123}
124
125#[derive(Debug, Clone, PartialEq, Eq)]
127pub struct RunnerRecord {
128 pub name: String,
130 pub repository: String,
132 pub profile: String,
134 pub slot: usize,
136 pub container: String,
138 pub started_at: u64,
140 pub memory_cap: Option<u64>,
142 pub cpu_cap: Option<u64>,
144 pub admission: Option<api::Admission>,
146 pub purpose: Option<&'static str>,
148}
149
150#[derive(Debug, Clone, PartialEq, Eq)]
152pub struct SampleRecord {
153 pub runner: String,
155 pub at: u64,
157 pub cpu: u64,
159 pub cpu_usage: u64,
161 pub memory: u64,
163 pub network_received: u64,
165 pub network_sent: u64,
167 pub events: Events,
169 pub cpu_held: CpuHeld,
171}
172
173#[derive(Debug, Clone, Default, PartialEq, Eq)]
175pub struct Measured {
176 pub at: Option<u64>,
178 pub bytes: BTreeMap<String, u64>,
180}
181
182#[derive(Debug, Clone, Default, PartialEq, Eq)]
185pub struct Announced {
186 pub profiles: Vec<api::ProfileStatus>,
188 pub repositories: Vec<api::RepositoryStatus>,
190}
191
192#[derive(Debug, Clone, PartialEq, Eq)]
194pub struct OpenJob {
195 pub id: u64,
197 pub repository: String,
199 pub state: api::JobState,
201 pub runner: Option<String>,
203 pub runner_removed_at: Option<u64>,
205 pub run_id: u64,
207 pub run_attempt: u32,
209 pub workflow: Option<String>,
211 pub queued_at: u64,
213 pub labels: Vec<String>,
215}
216
217#[derive(Debug, Clone, PartialEq, Eq)]
219pub struct SlotJob {
220 pub profile: String,
222 pub slot: usize,
224 pub key: crate::admission::Key,
226}
227
228#[derive(Debug, Clone, Copy, PartialEq, Eq)]
230pub struct LogPosition {
231 pub page: u64,
233 pub offset: u64,
235}
236
237#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
241pub struct MirrorPosition {
242 pub at: u64,
244 pub seen: u64,
246}
247
248#[derive(Debug)]
250enum Write {
251 Queued {
252 repository: String,
253 job: QueuedJob,
254 seen_at: u64,
255 },
256 RunnerStarted(RunnerRecord),
257 RunnerAdmission {
258 runner: String,
259 admission: api::Admission,
260 },
261 RunnerOomKilled(String),
262 RunnerStoppedFor {
263 runner: String,
264 reason: &'static str,
265 },
266 HostWaited {
267 jobs: Vec<u64>,
268 ms: u64,
269 },
270 SlotWaited {
271 jobs: Vec<u64>,
272 ms: u64,
273 },
274 BuildCache {
275 runner: String,
276 counts: api::BuildCacheCounts,
277 },
278 RunnerRemoved {
279 runner: String,
280 at: u64,
281 },
282 RunnersGone {
283 running: Vec<String>,
284 at: u64,
285 },
286 Claimed {
287 job: u64,
288 run_id: u64,
289 runner: String,
290 at: u64,
291 },
292 Details {
293 details: JobDetails,
294 },
295 Elsewhere {
296 details: JobDetails,
297 },
298 Lost {
299 job: u64,
300 at: u64,
301 },
302 Log {
303 job: u64,
304 lines: Vec<String>,
305 position: Option<LogPosition>,
307 max_bytes: u64,
308 _backlog: OwnedSemaphorePermit,
310 },
311 Samples(Vec<SampleRecord>),
312 HostSample(api::HostSample),
313 MirrorBytes {
314 bytes: Vec<(String, u64)>,
315 until: MirrorPosition,
316 done: oneshot::Sender<bool>,
318 },
319 CacheSizes {
320 at: u64,
321 sizes: BTreeMap<String, u64>,
322 },
323 Event(Box<Event>),
325 RecordMade(api::Record),
326 RecordSkipped {
327 job: u64,
328 why: String,
329 },
330 RecordRemoved {
331 job: u64,
332 at: u64,
333 by: api::RecordRemoval,
334 },
335 Paused {
336 repositories: Vec<String>,
337 done: oneshot::Sender<bool>,
339 },
340 Visibility {
341 repository: String,
342 visibility: api::Visibility,
343 at: u64,
344 },
345 SizingIntent {
346 change: api::SizingChange,
347 done: oneshot::Sender<Option<i64>>,
349 },
350 SizingResolved {
351 id: i64,
352 resolution: Resolution,
353 done: oneshot::Sender<bool>,
355 },
356 Notify {
357 change: NotifyChange,
358 done: oneshot::Sender<bool>,
360 },
361 Flush(oneshot::Sender<()>),
362}
363
364#[derive(Debug)]
366enum NotifyChange {
367 Record {
369 target: String,
370 set: Vec<(String, u64)>,
371 remove: Vec<String>,
372 },
373 Keep(Vec<String>),
375}
376
377#[derive(Debug, Clone, Copy, PartialEq, Eq)]
379pub enum Resolution {
380 Done,
382 Failed,
384 Uncertain,
387}
388
389#[derive(Debug, Clone, Default, PartialEq, Eq)]
391pub struct AutoSizing {
392 pub last: BTreeMap<String, u64>,
394 pub pending: Vec<(i64, api::SizingChange)>,
396 pub cpus: Vec<sizing::CpusSet>,
399 pub memory: Vec<sizing::MemorySet>,
401 pub slots: Vec<sizing::SlotsSet>,
404}
405
406#[derive(Debug, Clone)]
408pub struct Store {
409 writes: mpsc::UnboundedSender<Write>,
410 reader: Arc<Mutex<Connection>>,
411 pruner: Arc<Mutex<Connection>>,
413 closed: Arc<AtomicBool>,
415 held: Arc<Mutex<Vec<api::HostSample>>>,
418 latest_host: Arc<Mutex<Option<api::HostSample>>>,
420 latest_host_memory: Arc<Mutex<Option<api::HostMemory>>>,
422 latest_samples: Arc<Mutex<BTreeMap<String, SampleRecord>>>,
424 events: broadcast::Sender<Arc<EventEnvelope>>,
425 backlog: Arc<Semaphore>,
426}
427
428impl Store {
429 pub fn open(path: &Path) -> Result<Store, StoreError> {
434 Store::open_at(path, crate::time::now_ms())
435 }
436
437 fn open_at(path: &Path, now: u64) -> Result<Store, StoreError> {
439 if let Some(dir) = path.parent().filter(|dir| !dir.as_os_str().is_empty()) {
440 std::fs::DirBuilder::new()
441 .recursive(true)
442 .mode(0o700)
443 .create(dir)?;
444 }
445 std::fs::OpenOptions::new()
447 .create(true)
448 .append(true)
449 .mode(0o600)
450 .open(path)?;
451 let mut db = Connection::open(path)?;
452 configure(&db)?;
453 prepare(&mut db)?;
454 let downtime = Downtime::before(&mut db, now)?;
455 let reader = Connection::open_with_flags(
456 path,
457 OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX,
458 )?;
459 configure(&reader)?;
460 let pruner = Connection::open(path)?;
461 configure(&pruner)?;
462 let (events, _) = broadcast::channel(EVENT_BUFFER);
463 let (writes, receiver) = mpsc::unbounded_channel();
464 let thread_events = events.clone();
465 let held = Arc::new(Mutex::new(Vec::new()));
466 let thread_held = held.clone();
467 std::thread::Builder::new()
468 .name("runnerd-store".to_owned())
469 .spawn(move || write::run(db, downtime, receiver, &thread_events, &thread_held))?;
470 Ok(Store {
471 writes,
472 reader: Arc::new(Mutex::new(reader)),
473 pruner: Arc::new(Mutex::new(pruner)),
474 closed: Arc::new(AtomicBool::new(false)),
475 held,
476 latest_host: Arc::new(Mutex::new(None)),
477 latest_host_memory: Arc::new(Mutex::new(None)),
478 latest_samples: Arc::new(Mutex::new(BTreeMap::new())),
479 events,
480 backlog: Arc::new(Semaphore::new(
481 usize::try_from(LOG_BACKLOG).unwrap_or(usize::MAX),
482 )),
483 })
484 }
485
486 fn send(&self, write: Write) {
487 let _ = self.writes.send(write);
489 }
490
491 pub fn queued(&self, repository: &str, job: &QueuedJob, seen_at: u64) {
495 self.send(Write::Queued {
496 repository: repository.to_owned(),
497 job: job.clone(),
498 seen_at,
499 });
500 }
501
502 pub fn runner_started(&self, runner: RunnerRecord) {
504 self.send(Write::RunnerStarted(runner));
505 }
506
507 pub fn runner_removed(&self, runner: &str, at: u64) {
509 lock(&self.latest_samples).remove(runner);
510 self.send(Write::RunnerRemoved {
511 runner: runner.to_owned(),
512 at,
513 });
514 }
515
516 pub fn runners_gone(&self, running: Vec<String>, at: u64) {
519 lock(&self.latest_samples).retain(|runner, _| running.contains(runner));
520 self.send(Write::RunnersGone { running, at });
521 }
522
523 pub fn claimed(&self, job: u64, run_id: u64, runner: &str, at: u64) {
525 self.send(Write::Claimed {
526 job,
527 run_id,
528 runner: runner.to_owned(),
529 at,
530 });
531 }
532
533 pub fn details(&self, details: JobDetails) {
535 self.send(Write::Details { details });
536 }
537
538 pub fn elsewhere(&self, details: JobDetails) {
541 self.send(Write::Elsewhere { details });
542 }
543
544 pub fn lost(&self, job: u64, at: u64) {
546 self.send(Write::Lost { job, at });
547 }
548
549 pub async fn log(&self, job: u64, lines: Vec<String>, position: LogPosition, max_bytes: u64) {
552 let size = |line: &String| line.len() as u64 + 1;
553 let mut lines = lines.into_iter().map(|mut line| {
554 if size(&line) > LOG_BACKLOG {
556 let mut end = usize::try_from(LOG_BACKLOG / 2).unwrap_or(usize::MAX);
557 while !line.is_char_boundary(end) {
558 end -= 1;
559 }
560 line.truncate(end);
561 }
562 line
563 });
564 let mut next = lines.next();
565 loop {
566 let (mut part, mut bytes) = (Vec::new(), 0);
567 while let Some(line) = next.take_if(|line| bytes + size(line) <= LOG_BACKLOG) {
568 bytes += size(&line);
569 part.push(line);
570 next = lines.next();
571 }
572 let last = next.is_none();
573 let share = u32::try_from(bytes.max(1)).unwrap_or(u32::MAX);
574 let Ok(backlog) = self.backlog.clone().acquire_many_owned(share).await else {
576 return;
577 };
578 self.send(Write::Log {
579 job,
580 lines: part,
581 position: last.then_some(position),
582 max_bytes,
583 _backlog: backlog,
584 });
585 if last {
586 return;
587 }
588 }
589 }
590
591 pub fn samples(&self, samples: Vec<SampleRecord>) {
593 let mut latest = lock(&self.latest_samples);
594 for sample in &samples {
595 latest.insert(sample.runner.clone(), sample.clone());
596 }
597 drop(latest);
598 self.send(Write::Samples(samples));
599 }
600
601 pub fn host_sample(&self, sample: api::HostSample) {
604 *lock(&self.latest_host) = Some(sample.clone());
605 lock(&self.held).push(sample.clone());
606 self.send(Write::HostSample(sample));
607 }
608
609 pub fn latest_sample(&self, runner: &str) -> Option<SampleRecord> {
611 lock(&self.latest_samples).get(runner).cloned()
612 }
613
614 pub fn runner_admission(&self, runner: &str, admission: api::Admission) {
616 self.send(Write::RunnerAdmission {
617 runner: runner.to_owned(),
618 admission,
619 });
620 }
621
622 pub fn runner_oom_killed(&self, runner: &str) {
625 self.send(Write::RunnerOomKilled(runner.to_owned()));
626 }
627
628 pub fn runner_stopped_for(&self, runner: &str, reason: &'static str) {
630 self.send(Write::RunnerStoppedFor {
631 runner: runner.to_owned(),
632 reason,
633 });
634 }
635
636 pub fn build_cache(&self, runner: &str, counts: api::BuildCacheCounts) {
638 self.send(Write::BuildCache {
639 runner: runner.to_owned(),
640 counts,
641 });
642 }
643
644 pub fn host_memory(&self, memory: api::HostMemory) {
646 *lock(&self.latest_host_memory) = Some(memory);
647 }
648
649 pub fn latest_host_memory(&self) -> Option<api::HostMemory> {
651 lock(&self.latest_host_memory).clone()
652 }
653
654 pub fn host_waited(&self, jobs: Vec<u64>, ms: u64) {
657 self.send(Write::HostWaited { jobs, ms });
658 }
659
660 pub fn slot_waited(&self, jobs: Vec<u64>, ms: u64) {
663 self.send(Write::SlotWaited { jobs, ms });
664 }
665
666 pub fn latest_host_sample(&self) -> Option<api::HostSample> {
668 lock(&self.latest_host).clone()
669 }
670
671 pub async fn mirror_bytes(&self, bytes: Vec<(String, u64)>, until: MirrorPosition) -> bool {
674 let (done, committed) = oneshot::channel();
675 self.send(Write::MirrorBytes { bytes, until, done });
676 committed.await.unwrap_or(false)
677 }
678
679 pub fn cache_sizes(&self, at: u64, sizes: BTreeMap<String, u64>) {
681 self.send(Write::CacheSizes { at, sizes });
682 }
683
684 pub fn event(&self, event: Event) {
686 self.send(Write::Event(Box::new(event)));
687 }
688
689 pub fn record_made(&self, record: api::Record) {
691 self.send(Write::RecordMade(record));
692 }
693
694 pub fn record_skipped(&self, job: u64, why: String) {
696 self.send(Write::RecordSkipped { job, why });
697 }
698
699 pub fn record_removed(&self, job: u64, at: u64, by: api::RecordRemoval) {
701 self.send(Write::RecordRemoved { job, at, by });
702 }
703
704 pub fn paused(&self, repositories: Vec<String>) -> impl Future<Output = bool> + use<> {
707 let (done, committed) = oneshot::channel();
708 self.send(Write::Paused { repositories, done });
709 async move { committed.await.unwrap_or(false) }
710 }
711
712 pub fn visibility(&self, repository: String, visibility: api::Visibility, at: u64) {
714 self.send(Write::Visibility {
715 repository,
716 visibility,
717 at,
718 });
719 }
720
721 pub fn sizing_intent(
724 &self,
725 change: api::SizingChange,
726 ) -> impl Future<Output = Option<i64>> + use<> {
727 let (done, committed) = oneshot::channel();
728 self.send(Write::SizingIntent { change, done });
729 async move { committed.await.ok().flatten() }
730 }
731
732 pub fn sizing_resolved(
735 &self,
736 id: i64,
737 resolution: Resolution,
738 ) -> impl Future<Output = bool> + use<> {
739 let (done, committed) = oneshot::channel();
740 self.send(Write::SizingResolved {
741 id,
742 resolution,
743 done,
744 });
745 async move { committed.await.unwrap_or(false) }
746 }
747
748 pub fn notify_record(
751 &self,
752 target: &str,
753 set: Vec<(String, u64)>,
754 remove: Vec<String>,
755 ) -> impl Future<Output = bool> + use<> {
756 let (done, committed) = oneshot::channel();
757 self.send(Write::Notify {
758 change: NotifyChange::Record {
759 target: target.to_owned(),
760 set,
761 remove,
762 },
763 done,
764 });
765 async move { committed.await.unwrap_or(false) }
766 }
767
768 pub fn notify_keep(&self, targets: Vec<String>) -> impl Future<Output = bool> + use<> {
770 let (done, committed) = oneshot::channel();
771 self.send(Write::Notify {
772 change: NotifyChange::Keep(targets),
773 done,
774 });
775 async move { committed.await.unwrap_or(false) }
776 }
777
778 pub fn prune(&self, now: u64, history: &History) -> tokio::task::JoinHandle<()> {
782 let pruner = self.pruner.clone();
783 let closed = self.closed.clone();
784 let history = history.clone();
785 tokio::task::spawn_blocking(move || {
786 let db = match pruner.try_lock() {
787 Ok(db) => db,
788 Err(TryLockError::Poisoned(poisoned)) => poisoned.into_inner(),
789 Err(TryLockError::WouldBlock) => return,
790 };
791 let db = prune::Db {
792 connection: &db,
793 stop: &closed,
794 };
795 if !db.stop.load(Ordering::Relaxed) {
796 prune::run(&db, now, &history);
797 }
798 })
799 }
800
801 pub fn close(&self) {
804 self.closed.store(true, Ordering::Relaxed);
805 }
806
807 pub async fn flush(&self) {
809 let (done, wait) = oneshot::channel();
810 self.send(Write::Flush(done));
811 let _ = wait.await;
812 }
813
814 pub fn subscribe(&self) -> broadcast::Receiver<Arc<EventEnvelope>> {
817 self.events.subscribe()
818 }
819
820 async fn read<T: Send + 'static>(
821 &self,
822 query: impl FnOnce(&Connection) -> rusqlite::Result<T> + Send + 'static,
823 ) -> Result<T, StoreError> {
824 let reader = self.reader.clone();
825 Ok(tokio::task::spawn_blocking(move || query(&lock(&reader))).await??)
827 }
828
829 pub async fn jobs(&self, query: api::JobQuery) -> Result<api::JobPage, StoreError> {
831 self.read(move |db| read::jobs(db, &query)).await
832 }
833
834 pub async fn job(&self, id: u64) -> Result<Option<api::Job>, StoreError> {
836 self.read(move |db| read::job(db, id)).await
837 }
838
839 pub async fn job_samples(
841 &self,
842 id: u64,
843 interval: Duration,
844 ) -> Result<Option<api::SampleSeries>, StoreError> {
845 self.read(move |db| read::job_samples(db, id, interval))
846 .await
847 }
848
849 pub async fn log_lines(
851 &self,
852 job: u64,
853 from: u64,
854 limit: u64,
855 ) -> Result<Vec<(u64, String)>, StoreError> {
856 self.read(move |db| read::log_lines(db, job, from, limit))
857 .await
858 }
859
860 pub async fn events_after(
862 &self,
863 after: u64,
864 limit: u64,
865 ) -> Result<Vec<EventEnvelope>, StoreError> {
866 self.read(move |db| read::events_after(db, after, limit))
867 .await
868 }
869
870 pub async fn sizing_changes(&self, since: u64) -> Result<Vec<api::SizingChange>, StoreError> {
872 self.read(move |db| read::sizing_changes(db, since)).await
873 }
874
875 pub async fn auto_sizing(&self) -> Result<AutoSizing, StoreError> {
877 self.read(read::auto_sizing).await
878 }
879
880 pub fn announced(&self) -> Result<Announced, StoreError> {
883 Ok(read::announced(&lock(&self.reader))?)
884 }
885
886 pub fn paused_repositories(&self) -> Result<Vec<String>, StoreError> {
888 Ok(read::paused(&lock(&self.reader))?)
889 }
890
891 pub async fn host_samples(&self, since: u64) -> Result<Vec<api::HostSample>, StoreError> {
893 let held = lock(&self.held).clone();
896 let mut samples = self.read(move |db| read::host_samples(db, since)).await?;
897 let stored = samples.last().map_or(0, |sample| sample.at);
898 samples.extend(
899 held.into_iter()
900 .filter(|sample| sample.at >= since && sample.at > stored),
901 );
902 Ok(samples)
903 }
904
905 pub async fn activity(&self, since: u64, now: u64) -> Result<api::Activity, StoreError> {
907 self.read(move |db| read::activity(db, since, now)).await
908 }
909
910 pub async fn stats(&self, query: api::StatsQuery, now: u64) -> Result<api::Stats, StoreError> {
912 self.read(move |db| read::stats(db, &query, now)).await
913 }
914
915 pub async fn sizing(&self, since: u64) -> Result<sizing::History, StoreError> {
917 self.read(move |db| read::sizing(db, since)).await
918 }
919
920 pub async fn pricing(&self, since: u64) -> Result<api::PricingHistory, StoreError> {
923 self.read(move |db| read::pricing(db, since)).await
924 }
925
926 pub async fn insights(&self, since: u64) -> Result<crate::insights::History, StoreError> {
929 self.read(move |db| read::insights(db, since)).await
930 }
931
932 pub async fn last_event(&self) -> Result<u64, StoreError> {
934 self.read(read::last_event).await
935 }
936
937 pub async fn notify_state(
939 &self,
940 ) -> Result<BTreeMap<String, BTreeMap<String, u64>>, StoreError> {
941 self.read(read::notify_state).await
942 }
943
944 pub async fn warm(&self, since: u64) -> Result<Vec<warm::Seen>, StoreError> {
946 self.read(move |db| read::warm(db, since)).await
947 }
948
949 pub async fn cache_sizes_now(&self) -> Result<Measured, StoreError> {
951 self.read(read::latest_cache_sizes).await
952 }
953
954 pub async fn build_cache_since(&self, since: u64) -> Result<api::BuildCacheCounts, StoreError> {
956 self.read(move |db| read::build_cache_since(db, since))
957 .await
958 }
959
960 pub async fn cache_history(&self, since: u64) -> Result<Vec<api::CacheSize>, StoreError> {
962 self.read(move |db| read::cache_history(db, since)).await
963 }
964
965 pub async fn open_jobs(&self) -> Result<Vec<OpenJob>, StoreError> {
967 self.read(read::open_jobs).await
968 }
969
970 pub async fn slot_jobs(&self) -> Result<Vec<SlotJob>, StoreError> {
972 self.read(read::slot_jobs).await
973 }
974
975 pub async fn last_runner_second(&self, prefix: String) -> Result<Option<u64>, StoreError> {
978 self.read(move |db| read::last_runner_second(db, &prefix))
979 .await
980 }
981
982 pub async fn runner_job(&self, runner: &str) -> Result<Option<u64>, StoreError> {
984 let runner = runner.to_owned();
985 self.read(move |db| read::runner_job(db, &runner)).await
986 }
987
988 pub async fn runner_admitted(
990 &self,
991 runner: &str,
992 ) -> Result<Option<api::Admission>, StoreError> {
993 let runner = runner.to_owned();
994 self.read(move |db| read::runner_admission(db, &runner))
995 .await
996 }
997
998 pub async fn log_position(&self, job: u64) -> Result<LogPosition, StoreError> {
1000 self.read(move |db| read::log_position(db, job)).await
1001 }
1002
1003 pub async fn mirror_log_until(&self) -> Result<MirrorPosition, StoreError> {
1005 self.read(read::mirror_log_until).await
1006 }
1007}
1008
1009pub fn sizing_at(path: &Path, since: u64) -> Result<Option<sizing::History>, StoreError> {
1014 if !path.exists() {
1015 return Ok(None);
1016 }
1017 let read = |path: &str, flags| -> Result<sizing::History, StoreError> {
1018 let db = Connection::open_with_flags(path, flags)?;
1019 db.busy_timeout(Duration::from_secs(5))?;
1020 sizing_of(&db, since)
1021 };
1022 let name = path.to_string_lossy();
1023 match read(&name, OpenFlags::SQLITE_OPEN_READ_ONLY) {
1024 Err(StoreError::Sqlite(error))
1025 if matches!(
1026 error.sqlite_error_code(),
1027 Some(rusqlite::ErrorCode::ReadOnly | rusqlite::ErrorCode::CannotOpen)
1028 ) =>
1029 {
1030 let mut wal = path.as_os_str().to_owned();
1032 wal.push("-wal");
1033 let wal = PathBuf::from(wal);
1034 if std::fs::metadata(&wal).is_ok_and(|wal| wal.len() > 0) {
1035 return Err(StoreError::Unread(wal));
1036 }
1037 let uri = format!(
1038 "file:{}?mode=ro&immutable=1",
1039 name.replace('%', "%25")
1040 .replace('?', "%3f")
1041 .replace('#', "%23")
1042 );
1043 read(
1044 &uri,
1045 OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_URI,
1046 )
1047 }
1048 history => history,
1049 }
1050 .map(Some)
1051}
1052
1053fn sizing_of(db: &Connection, since: u64) -> Result<sizing::History, StoreError> {
1055 match version(db)? {
1056 0 => return Ok(sizing::History::default()),
1057 VERSION => {}
1058 _ => read_as_current(db)?,
1059 }
1060 let mut history = read::sizing(db, since)?;
1061 let auto = read::auto_sizing(db)?;
1062 history.memory_set = auto.memory;
1063 history.slots_set = auto.slots;
1064 history.changes = read::sizing_changes(db, since.saturating_sub(sizing::memory::HISTORY))?;
1065 Ok(history)
1066}
1067
1068fn version(db: &Connection) -> Result<u32, StoreError> {
1070 let found: u32 = db.pragma_query_value(None, "user_version", |row| row.get(0))?;
1071 match found {
1072 0 | OLDEST..=VERSION => Ok(found),
1073 found if found > VERSION => Err(StoreError::Newer {
1074 found,
1075 supported: VERSION,
1076 }),
1077 found => Err(StoreError::Older {
1078 found,
1079 oldest: OLDEST,
1080 }),
1081 }
1082}
1083
1084fn read_as_current(db: &Connection) -> Result<(), StoreError> {
1088 let current = Connection::open_in_memory()?;
1089 current.execute_batch(SCHEMA)?;
1090 let tables: Vec<(String, String)> = current
1091 .prepare(
1092 "SELECT name, sql FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'",
1093 )?
1094 .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?
1095 .collect::<rusqlite::Result<_>>()?;
1096 let columns =
1097 |db: &Connection, table: &str| -> rusqlite::Result<Vec<(String, Option<String>)>> {
1098 db.prepare("SELECT name, dflt_value FROM pragma_table_info(?1, 'main')")?
1099 .query_map([table], |row| Ok((row.get(0)?, row.get(1)?)))?
1100 .collect()
1101 };
1102 for (table, sql) in tables {
1103 let found: Vec<String> = columns(db, &table)?
1104 .into_iter()
1105 .map(|(name, _)| name)
1106 .collect();
1107 if found.is_empty() {
1108 db.execute_batch(&sql.replacen("CREATE TABLE", "CREATE TEMP TABLE", 1))?;
1109 continue;
1110 }
1111 let added: Vec<String> = columns(¤t, &table)?
1112 .into_iter()
1113 .filter(|(name, _)| !found.contains(name))
1114 .map(|(name, default)| format!("{} AS {name}", default.as_deref().unwrap_or("NULL")))
1115 .collect();
1116 if !added.is_empty() {
1117 db.execute_batch(&format!(
1118 "CREATE TEMP VIEW {table} AS SELECT *, {} FROM main.{table}",
1119 added.join(", ")
1120 ))?;
1121 }
1122 }
1123 Ok(())
1124}
1125
1126fn configure(db: &Connection) -> rusqlite::Result<()> {
1128 db.pragma_update(None, "journal_mode", "wal")?;
1129 db.pragma_update(None, "synchronous", "normal")?;
1130 db.pragma_update(None, "foreign_keys", "on")?;
1131 db.busy_timeout(Duration::from_secs(5))
1132}
1133
1134fn prepare(db: &mut Connection) -> Result<(), StoreError> {
1137 let found = version(db)?;
1138 if found == 0 {
1139 let transaction = db.transaction()?;
1140 transaction.execute_batch(SCHEMA)?;
1141 transaction.pragma_update(None, "user_version", VERSION)?;
1142 transaction.commit()?;
1143 return Ok(());
1144 }
1145 for (version, migration) in MIGRATIONS.iter().filter(|(version, _)| *version > found) {
1146 let transaction = db.transaction()?;
1147 transaction.execute_batch(migration)?;
1148 transaction.pragma_update(None, "user_version", version)?;
1149 transaction.commit()?;
1150 }
1151 Ok(())
1152}
1153
1154#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1159struct Downtime {
1160 from: u64,
1161 until: u64,
1162}
1163
1164impl Downtime {
1165 fn before(db: &mut Connection, now: u64) -> Result<Option<Downtime>, StoreError> {
1169 let transaction = db.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?;
1172 let last: Option<u64> = transaction.query_row(
1173 "SELECT max(at) FROM (SELECT max(at) AS at FROM host_samples UNION ALL SELECT max(at) FROM events \
1174 UNION ALL SELECT value FROM meta WHERE key = 'opened_at')",
1175 [],
1176 |row| row.get(0),
1177 )?;
1178 let downtime = last
1179 .filter(|last| *last < now)
1180 .map(|from| Downtime { from, until: now });
1181 if let Some(downtime) = downtime {
1182 let queued: Vec<(u64, u64)> = transaction
1183 .prepare("SELECT id, queued_at FROM jobs WHERE state = 'queued'")?
1184 .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?
1185 .collect::<rusqlite::Result<_>>()?;
1186 for (job, queued_at) in queued {
1187 transaction.execute(
1188 "UPDATE jobs SET unseen = unseen + ?2 WHERE id = ?1",
1189 rusqlite::params![job, downtime.of(queued_at)],
1190 )?;
1191 }
1192 }
1193 transaction.execute(
1194 "INSERT OR REPLACE INTO meta (key, value) VALUES ('opened_at', ?1)",
1195 [now],
1196 )?;
1197 transaction.commit()?;
1198 Ok(downtime)
1199 }
1200
1201 fn of(self, queued_at: u64) -> u64 {
1203 self.until.saturating_sub(queued_at.max(self.from))
1204 }
1205
1206 fn after(self, queued_at: u64, started_at: u64) -> u64 {
1209 self.of(queued_at)
1210 .saturating_sub(started_at.saturating_sub(queued_at.max(self.from)))
1211 }
1212}
1213
1214fn logged<T>(what: &str, result: rusqlite::Result<T>) -> Option<T> {
1216 result
1217 .map_err(|error| warn!("job history: cannot {what}: {error}"))
1218 .ok()
1219}
1220
1221#[cfg(test)]
1222mod tests;