1use std::collections::BTreeMap;
8use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt};
9use std::path::{Path, PathBuf};
10use std::sync::atomic::{AtomicBool, Ordering};
11use std::sync::{Arc, Mutex, TryLockError};
12use std::time::Duration;
13
14use rusqlite::{Connection, OpenFlags};
15use tokio::sync::{OwnedSemaphorePermit, Semaphore, broadcast, mpsc, oneshot};
16use tracing::warn;
17
18use crate::api::{self, Event, EventEnvelope};
19use crate::config::History;
20use crate::github::{JobDetails, QueuedJob};
21use crate::observe::metrics::{CpuHeld, Events};
22use crate::scheduler::warm;
23use crate::sizing;
24use crate::sync::lock;
25
26mod prune;
27mod read;
28mod write;
29
30const SCHEMA: &str = include_str!("store/schema.sql");
32
33const OLDEST: u32 = 15;
35
36const MIGRATIONS: &[(u32, &str)] = &[
39 (
40 16,
41 "ALTER TABLE jobs ADD COLUMN unseen INTEGER NOT NULL DEFAULT 0;",
42 ),
43 (
44 17,
45 "ALTER TABLE jobs ADD COLUMN cpu_periods INTEGER;\n\
46 ALTER TABLE jobs ADD COLUMN cpu_throttled INTEGER;\n\
47 ALTER TABLE jobs ADD COLUMN cpu_stalled INTEGER;\n\
48 ALTER TABLE samples ADD COLUMN cpu_periods INTEGER;\n\
49 ALTER TABLE samples ADD COLUMN cpu_throttled INTEGER;\n\
50 ALTER TABLE samples ADD COLUMN cpu_stalled INTEGER;",
51 ),
52 (
53 18,
54 "ALTER TABLE jobs ADD COLUMN slot_waited INTEGER NOT NULL DEFAULT 0;",
55 ),
56];
57
58const VERSION: u32 = match MIGRATIONS.last() {
60 Some((version, _)) => *version,
61 None => OLDEST,
62};
63
64pub(crate) const LOG_BACKLOG: u64 = 8 << 20;
67
68const HOST_SAMPLE_BATCH: usize = 30;
71
72const EVENT_BUFFER: usize = 4096;
74
75#[derive(Debug, thiserror::Error)]
77pub enum StoreError {
78 #[error("{0}")]
80 Sqlite(#[from] rusqlite::Error),
81 #[error(
83 "the job history database is version {found}, newer than the {supported} this runnerd reads; \
84 run a newer runnerd or move the database aside"
85 )]
86 Newer {
87 found: u32,
89 supported: u32,
91 },
92 #[error(
94 "the job history database is version {found}, from a runnerd before 0.0.19; this runnerd reads versions \
95 from {oldest} on: move the database aside"
96 )]
97 Older {
98 found: u32,
100 oldest: u32,
102 },
103 #[error("{0}")]
105 Io(#[from] std::io::Error),
106 #[error("a database read stopped: {0}")]
108 Join(#[from] tokio::task::JoinError),
109 #[error(
112 "{} holds changes not yet in the job history, which this user cannot read; run runnerd doctor as the user \
113 the daemon runs as",
114 .0.display()
115 )]
116 Unread(PathBuf),
117}
118
119#[derive(Debug, Clone, PartialEq, Eq)]
121pub struct RunnerRecord {
122 pub name: String,
124 pub repository: String,
126 pub profile: String,
128 pub slot: usize,
130 pub container: String,
132 pub started_at: u64,
134 pub memory_cap: Option<u64>,
136 pub cpu_cap: Option<u64>,
138 pub admission: Option<api::Admission>,
140}
141
142#[derive(Debug, Clone, PartialEq, Eq)]
144pub struct SampleRecord {
145 pub runner: String,
147 pub at: u64,
149 pub cpu: u64,
151 pub cpu_usage: u64,
153 pub memory: u64,
155 pub network_received: u64,
157 pub network_sent: u64,
159 pub events: Events,
161 pub cpu_held: CpuHeld,
163}
164
165#[derive(Debug, Clone, Default, PartialEq, Eq)]
167pub struct Measured {
168 pub at: Option<u64>,
170 pub bytes: BTreeMap<String, u64>,
172}
173
174#[derive(Debug, Clone, Default, PartialEq, Eq)]
177pub struct Announced {
178 pub profiles: Vec<api::ProfileStatus>,
180 pub repositories: Vec<api::RepositoryStatus>,
182}
183
184#[derive(Debug, Clone, PartialEq, Eq)]
186pub struct OpenJob {
187 pub id: u64,
189 pub repository: String,
191 pub state: api::JobState,
193 pub runner: Option<String>,
195 pub runner_removed_at: Option<u64>,
197 pub run_id: u64,
199 pub run_attempt: u32,
201 pub workflow: Option<String>,
203 pub queued_at: u64,
205 pub labels: Vec<String>,
207}
208
209#[derive(Debug, Clone, PartialEq, Eq)]
211pub struct SlotJob {
212 pub profile: String,
214 pub slot: usize,
216 pub key: crate::admission::Key,
218}
219
220#[derive(Debug, Clone, Copy, PartialEq, Eq)]
222pub struct LogPosition {
223 pub page: u64,
225 pub offset: u64,
227}
228
229#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
233pub struct MirrorPosition {
234 pub at: u64,
236 pub seen: u64,
238}
239
240#[derive(Debug)]
242enum Write {
243 Queued {
244 repository: String,
245 job: QueuedJob,
246 },
247 RunnerStarted(RunnerRecord),
248 RunnerAdmission {
249 runner: String,
250 admission: api::Admission,
251 },
252 RunnerOomKilled(String),
253 HostWaited {
254 jobs: Vec<u64>,
255 ms: u64,
256 },
257 SlotWaited {
258 jobs: Vec<u64>,
259 ms: u64,
260 },
261 BuildCache {
262 runner: String,
263 counts: api::BuildCacheCounts,
264 },
265 RunnerRemoved {
266 runner: String,
267 at: u64,
268 },
269 RunnersGone {
270 running: Vec<String>,
271 at: u64,
272 },
273 Claimed {
274 job: u64,
275 run_id: u64,
276 runner: String,
277 at: u64,
278 },
279 Details {
280 details: JobDetails,
281 },
282 Elsewhere {
283 details: JobDetails,
284 },
285 Lost {
286 job: u64,
287 at: u64,
288 },
289 Log {
290 job: u64,
291 lines: Vec<String>,
292 position: Option<LogPosition>,
294 max_bytes: u64,
295 _backlog: OwnedSemaphorePermit,
297 },
298 Samples(Vec<SampleRecord>),
299 HostSample(api::HostSample),
300 MirrorBytes {
301 bytes: Vec<(String, u64)>,
302 until: MirrorPosition,
303 done: oneshot::Sender<bool>,
305 },
306 CacheSizes {
307 at: u64,
308 sizes: BTreeMap<String, u64>,
309 },
310 Event(Box<Event>),
312 RecordMade(api::Record),
313 RecordSkipped {
314 job: u64,
315 why: String,
316 },
317 RecordRemoved {
318 job: u64,
319 at: u64,
320 by: api::RecordRemoval,
321 },
322 Paused {
323 repositories: Vec<String>,
324 done: oneshot::Sender<bool>,
326 },
327 Visibility {
328 repository: String,
329 visibility: api::Visibility,
330 at: u64,
331 },
332 SizingIntent {
333 change: api::SizingChange,
334 done: oneshot::Sender<Option<i64>>,
336 },
337 SizingResolved {
338 id: i64,
339 resolution: Resolution,
340 done: oneshot::Sender<bool>,
342 },
343 Notify {
344 change: NotifyChange,
345 done: oneshot::Sender<bool>,
347 },
348 Flush(oneshot::Sender<()>),
349}
350
351#[derive(Debug)]
353enum NotifyChange {
354 Record {
356 target: String,
357 set: Vec<(String, u64)>,
358 remove: Vec<String>,
359 },
360 Keep(Vec<String>),
362}
363
364#[derive(Debug, Clone, Copy, PartialEq, Eq)]
366pub enum Resolution {
367 Done,
369 Failed,
371 Uncertain,
374}
375
376#[derive(Debug, Clone, Default, PartialEq, Eq)]
378pub struct AutoSizing {
379 pub last: BTreeMap<String, u64>,
381 pub pending: Vec<(i64, api::SizingChange)>,
383 pub cpus: Vec<sizing::CpusSet>,
386 pub memory: Vec<sizing::MemorySet>,
388 pub slots: Vec<sizing::SlotsSet>,
391}
392
393#[derive(Debug, Clone)]
395pub struct Store {
396 writes: mpsc::UnboundedSender<Write>,
397 reader: Arc<Mutex<Connection>>,
398 pruner: Arc<Mutex<Connection>>,
400 closed: Arc<AtomicBool>,
402 held: Arc<Mutex<Vec<api::HostSample>>>,
405 latest_host: Arc<Mutex<Option<api::HostSample>>>,
407 latest_host_memory: Arc<Mutex<Option<api::HostMemory>>>,
409 latest_samples: Arc<Mutex<BTreeMap<String, SampleRecord>>>,
411 events: broadcast::Sender<Arc<EventEnvelope>>,
412 backlog: Arc<Semaphore>,
413}
414
415impl Store {
416 pub fn open(path: &Path) -> Result<Store, StoreError> {
421 Store::open_at(path, crate::time::now_ms())
422 }
423
424 fn open_at(path: &Path, now: u64) -> Result<Store, StoreError> {
426 if let Some(dir) = path.parent().filter(|dir| !dir.as_os_str().is_empty()) {
427 std::fs::DirBuilder::new()
428 .recursive(true)
429 .mode(0o700)
430 .create(dir)?;
431 }
432 std::fs::OpenOptions::new()
434 .create(true)
435 .append(true)
436 .mode(0o600)
437 .open(path)?;
438 let mut db = Connection::open(path)?;
439 configure(&db)?;
440 prepare(&mut db)?;
441 let downtime = Downtime::before(&mut db, now)?;
442 let reader = Connection::open_with_flags(
443 path,
444 OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX,
445 )?;
446 configure(&reader)?;
447 let pruner = Connection::open(path)?;
448 configure(&pruner)?;
449 let (events, _) = broadcast::channel(EVENT_BUFFER);
450 let (writes, receiver) = mpsc::unbounded_channel();
451 let thread_events = events.clone();
452 let held = Arc::new(Mutex::new(Vec::new()));
453 let thread_held = held.clone();
454 std::thread::Builder::new()
455 .name("runnerd-store".to_owned())
456 .spawn(move || write::run(db, downtime, receiver, &thread_events, &thread_held))?;
457 Ok(Store {
458 writes,
459 reader: Arc::new(Mutex::new(reader)),
460 pruner: Arc::new(Mutex::new(pruner)),
461 closed: Arc::new(AtomicBool::new(false)),
462 held,
463 latest_host: Arc::new(Mutex::new(None)),
464 latest_host_memory: Arc::new(Mutex::new(None)),
465 latest_samples: Arc::new(Mutex::new(BTreeMap::new())),
466 events,
467 backlog: Arc::new(Semaphore::new(
468 usize::try_from(LOG_BACKLOG).unwrap_or(usize::MAX),
469 )),
470 })
471 }
472
473 fn send(&self, write: Write) {
474 let _ = self.writes.send(write);
476 }
477
478 pub fn queued(&self, repository: &str, job: &QueuedJob) {
480 self.send(Write::Queued {
481 repository: repository.to_owned(),
482 job: job.clone(),
483 });
484 }
485
486 pub fn runner_started(&self, runner: RunnerRecord) {
488 self.send(Write::RunnerStarted(runner));
489 }
490
491 pub fn runner_removed(&self, runner: &str, at: u64) {
493 lock(&self.latest_samples).remove(runner);
494 self.send(Write::RunnerRemoved {
495 runner: runner.to_owned(),
496 at,
497 });
498 }
499
500 pub fn runners_gone(&self, running: Vec<String>, at: u64) {
503 lock(&self.latest_samples).retain(|runner, _| running.contains(runner));
504 self.send(Write::RunnersGone { running, at });
505 }
506
507 pub fn claimed(&self, job: u64, run_id: u64, runner: &str, at: u64) {
509 self.send(Write::Claimed {
510 job,
511 run_id,
512 runner: runner.to_owned(),
513 at,
514 });
515 }
516
517 pub fn details(&self, details: JobDetails) {
519 self.send(Write::Details { details });
520 }
521
522 pub fn elsewhere(&self, details: JobDetails) {
525 self.send(Write::Elsewhere { details });
526 }
527
528 pub fn lost(&self, job: u64, at: u64) {
530 self.send(Write::Lost { job, at });
531 }
532
533 pub async fn log(&self, job: u64, lines: Vec<String>, position: LogPosition, max_bytes: u64) {
536 let size = |line: &String| line.len() as u64 + 1;
537 let mut lines = lines.into_iter().map(|mut line| {
538 if size(&line) > LOG_BACKLOG {
540 let mut end = usize::try_from(LOG_BACKLOG / 2).unwrap_or(usize::MAX);
541 while !line.is_char_boundary(end) {
542 end -= 1;
543 }
544 line.truncate(end);
545 }
546 line
547 });
548 let mut next = lines.next();
549 loop {
550 let (mut part, mut bytes) = (Vec::new(), 0);
551 while let Some(line) = next.take_if(|line| bytes + size(line) <= LOG_BACKLOG) {
552 bytes += size(&line);
553 part.push(line);
554 next = lines.next();
555 }
556 let last = next.is_none();
557 let share = u32::try_from(bytes.max(1)).unwrap_or(u32::MAX);
558 let Ok(backlog) = self.backlog.clone().acquire_many_owned(share).await else {
560 return;
561 };
562 self.send(Write::Log {
563 job,
564 lines: part,
565 position: last.then_some(position),
566 max_bytes,
567 _backlog: backlog,
568 });
569 if last {
570 return;
571 }
572 }
573 }
574
575 pub fn samples(&self, samples: Vec<SampleRecord>) {
577 let mut latest = lock(&self.latest_samples);
578 for sample in &samples {
579 latest.insert(sample.runner.clone(), sample.clone());
580 }
581 drop(latest);
582 self.send(Write::Samples(samples));
583 }
584
585 pub fn host_sample(&self, sample: api::HostSample) {
588 *lock(&self.latest_host) = Some(sample.clone());
589 lock(&self.held).push(sample.clone());
590 self.send(Write::HostSample(sample));
591 }
592
593 pub fn latest_sample(&self, runner: &str) -> Option<SampleRecord> {
595 lock(&self.latest_samples).get(runner).cloned()
596 }
597
598 pub fn runner_admission(&self, runner: &str, admission: api::Admission) {
600 self.send(Write::RunnerAdmission {
601 runner: runner.to_owned(),
602 admission,
603 });
604 }
605
606 pub fn runner_oom_killed(&self, runner: &str) {
609 self.send(Write::RunnerOomKilled(runner.to_owned()));
610 }
611
612 pub fn build_cache(&self, runner: &str, counts: api::BuildCacheCounts) {
614 self.send(Write::BuildCache {
615 runner: runner.to_owned(),
616 counts,
617 });
618 }
619
620 pub fn host_memory(&self, memory: api::HostMemory) {
622 *lock(&self.latest_host_memory) = Some(memory);
623 }
624
625 pub fn latest_host_memory(&self) -> Option<api::HostMemory> {
627 lock(&self.latest_host_memory).clone()
628 }
629
630 pub fn host_waited(&self, jobs: Vec<u64>, ms: u64) {
633 self.send(Write::HostWaited { jobs, ms });
634 }
635
636 pub fn slot_waited(&self, jobs: Vec<u64>, ms: u64) {
639 self.send(Write::SlotWaited { jobs, ms });
640 }
641
642 pub fn latest_host_sample(&self) -> Option<api::HostSample> {
644 lock(&self.latest_host).clone()
645 }
646
647 pub async fn mirror_bytes(&self, bytes: Vec<(String, u64)>, until: MirrorPosition) -> bool {
650 let (done, committed) = oneshot::channel();
651 self.send(Write::MirrorBytes { bytes, until, done });
652 committed.await.unwrap_or(false)
653 }
654
655 pub fn cache_sizes(&self, at: u64, sizes: BTreeMap<String, u64>) {
657 self.send(Write::CacheSizes { at, sizes });
658 }
659
660 pub fn event(&self, event: Event) {
662 self.send(Write::Event(Box::new(event)));
663 }
664
665 pub fn record_made(&self, record: api::Record) {
667 self.send(Write::RecordMade(record));
668 }
669
670 pub fn record_skipped(&self, job: u64, why: String) {
672 self.send(Write::RecordSkipped { job, why });
673 }
674
675 pub fn record_removed(&self, job: u64, at: u64, by: api::RecordRemoval) {
677 self.send(Write::RecordRemoved { job, at, by });
678 }
679
680 pub fn paused(&self, repositories: Vec<String>) -> impl Future<Output = bool> + use<> {
683 let (done, committed) = oneshot::channel();
684 self.send(Write::Paused { repositories, done });
685 async move { committed.await.unwrap_or(false) }
686 }
687
688 pub fn visibility(&self, repository: String, visibility: api::Visibility, at: u64) {
690 self.send(Write::Visibility {
691 repository,
692 visibility,
693 at,
694 });
695 }
696
697 pub fn sizing_intent(
700 &self,
701 change: api::SizingChange,
702 ) -> impl Future<Output = Option<i64>> + use<> {
703 let (done, committed) = oneshot::channel();
704 self.send(Write::SizingIntent { change, done });
705 async move { committed.await.ok().flatten() }
706 }
707
708 pub fn sizing_resolved(
711 &self,
712 id: i64,
713 resolution: Resolution,
714 ) -> impl Future<Output = bool> + use<> {
715 let (done, committed) = oneshot::channel();
716 self.send(Write::SizingResolved {
717 id,
718 resolution,
719 done,
720 });
721 async move { committed.await.unwrap_or(false) }
722 }
723
724 pub fn notify_record(
727 &self,
728 target: &str,
729 set: Vec<(String, u64)>,
730 remove: Vec<String>,
731 ) -> impl Future<Output = bool> + use<> {
732 let (done, committed) = oneshot::channel();
733 self.send(Write::Notify {
734 change: NotifyChange::Record {
735 target: target.to_owned(),
736 set,
737 remove,
738 },
739 done,
740 });
741 async move { committed.await.unwrap_or(false) }
742 }
743
744 pub fn notify_keep(&self, targets: Vec<String>) -> impl Future<Output = bool> + use<> {
746 let (done, committed) = oneshot::channel();
747 self.send(Write::Notify {
748 change: NotifyChange::Keep(targets),
749 done,
750 });
751 async move { committed.await.unwrap_or(false) }
752 }
753
754 pub fn prune(&self, now: u64, history: &History) -> tokio::task::JoinHandle<()> {
758 let pruner = self.pruner.clone();
759 let closed = self.closed.clone();
760 let history = history.clone();
761 tokio::task::spawn_blocking(move || {
762 let db = match pruner.try_lock() {
763 Ok(db) => db,
764 Err(TryLockError::Poisoned(poisoned)) => poisoned.into_inner(),
765 Err(TryLockError::WouldBlock) => return,
766 };
767 let db = prune::Db {
768 connection: &db,
769 stop: &closed,
770 };
771 if !db.stop.load(Ordering::Relaxed) {
772 prune::run(&db, now, &history);
773 }
774 })
775 }
776
777 pub fn close(&self) {
780 self.closed.store(true, Ordering::Relaxed);
781 }
782
783 pub async fn flush(&self) {
785 let (done, wait) = oneshot::channel();
786 self.send(Write::Flush(done));
787 let _ = wait.await;
788 }
789
790 pub fn subscribe(&self) -> broadcast::Receiver<Arc<EventEnvelope>> {
793 self.events.subscribe()
794 }
795
796 async fn read<T: Send + 'static>(
797 &self,
798 query: impl FnOnce(&Connection) -> rusqlite::Result<T> + Send + 'static,
799 ) -> Result<T, StoreError> {
800 let reader = self.reader.clone();
801 Ok(tokio::task::spawn_blocking(move || query(&lock(&reader))).await??)
803 }
804
805 pub async fn jobs(&self, query: api::JobQuery) -> Result<api::JobPage, StoreError> {
807 self.read(move |db| read::jobs(db, &query)).await
808 }
809
810 pub async fn job(&self, id: u64) -> Result<Option<api::Job>, StoreError> {
812 self.read(move |db| read::job(db, id)).await
813 }
814
815 pub async fn job_samples(
817 &self,
818 id: u64,
819 interval: Duration,
820 ) -> Result<Option<api::SampleSeries>, StoreError> {
821 self.read(move |db| read::job_samples(db, id, interval))
822 .await
823 }
824
825 pub async fn log_lines(
827 &self,
828 job: u64,
829 from: u64,
830 limit: u64,
831 ) -> Result<Vec<(u64, String)>, StoreError> {
832 self.read(move |db| read::log_lines(db, job, from, limit))
833 .await
834 }
835
836 pub async fn events_after(
838 &self,
839 after: u64,
840 limit: u64,
841 ) -> Result<Vec<EventEnvelope>, StoreError> {
842 self.read(move |db| read::events_after(db, after, limit))
843 .await
844 }
845
846 pub async fn sizing_changes(&self, since: u64) -> Result<Vec<api::SizingChange>, StoreError> {
848 self.read(move |db| read::sizing_changes(db, since)).await
849 }
850
851 pub async fn auto_sizing(&self) -> Result<AutoSizing, StoreError> {
853 self.read(read::auto_sizing).await
854 }
855
856 pub fn announced(&self) -> Result<Announced, StoreError> {
859 Ok(read::announced(&lock(&self.reader))?)
860 }
861
862 pub fn paused_repositories(&self) -> Result<Vec<String>, StoreError> {
864 Ok(read::paused(&lock(&self.reader))?)
865 }
866
867 pub async fn host_samples(&self, since: u64) -> Result<Vec<api::HostSample>, StoreError> {
869 let held = lock(&self.held).clone();
872 let mut samples = self.read(move |db| read::host_samples(db, since)).await?;
873 let stored = samples.last().map_or(0, |sample| sample.at);
874 samples.extend(
875 held.into_iter()
876 .filter(|sample| sample.at >= since && sample.at > stored),
877 );
878 Ok(samples)
879 }
880
881 pub async fn activity(&self, since: u64, now: u64) -> Result<api::Activity, StoreError> {
883 self.read(move |db| read::activity(db, since, now)).await
884 }
885
886 pub async fn stats(&self, query: api::StatsQuery, now: u64) -> Result<api::Stats, StoreError> {
888 self.read(move |db| read::stats(db, &query, now)).await
889 }
890
891 pub async fn sizing(&self, since: u64) -> Result<sizing::History, StoreError> {
893 self.read(move |db| read::sizing(db, since)).await
894 }
895
896 pub async fn pricing(&self, since: u64) -> Result<api::PricingHistory, StoreError> {
899 self.read(move |db| read::pricing(db, since)).await
900 }
901
902 pub async fn insights(&self, since: u64) -> Result<crate::insights::History, StoreError> {
905 self.read(move |db| read::insights(db, since)).await
906 }
907
908 pub async fn last_event(&self) -> Result<u64, StoreError> {
910 self.read(read::last_event).await
911 }
912
913 pub async fn notify_state(
915 &self,
916 ) -> Result<BTreeMap<String, BTreeMap<String, u64>>, StoreError> {
917 self.read(read::notify_state).await
918 }
919
920 pub async fn warm(&self, since: u64) -> Result<Vec<warm::Seen>, StoreError> {
922 self.read(move |db| read::warm(db, since)).await
923 }
924
925 pub async fn cache_sizes_now(&self) -> Result<Measured, StoreError> {
927 self.read(read::latest_cache_sizes).await
928 }
929
930 pub async fn build_cache_since(&self, since: u64) -> Result<api::BuildCacheCounts, StoreError> {
932 self.read(move |db| read::build_cache_since(db, since))
933 .await
934 }
935
936 pub async fn cache_history(&self, since: u64) -> Result<Vec<api::CacheSize>, StoreError> {
938 self.read(move |db| read::cache_history(db, since)).await
939 }
940
941 pub async fn open_jobs(&self) -> Result<Vec<OpenJob>, StoreError> {
943 self.read(read::open_jobs).await
944 }
945
946 pub async fn slot_jobs(&self) -> Result<Vec<SlotJob>, StoreError> {
948 self.read(read::slot_jobs).await
949 }
950
951 pub async fn runner_job(&self, runner: &str) -> Result<Option<u64>, StoreError> {
953 let runner = runner.to_owned();
954 self.read(move |db| read::runner_job(db, &runner)).await
955 }
956
957 pub async fn runner_admitted(
959 &self,
960 runner: &str,
961 ) -> Result<Option<api::Admission>, StoreError> {
962 let runner = runner.to_owned();
963 self.read(move |db| read::runner_admission(db, &runner))
964 .await
965 }
966
967 pub async fn log_position(&self, job: u64) -> Result<LogPosition, StoreError> {
969 self.read(move |db| read::log_position(db, job)).await
970 }
971
972 pub async fn mirror_log_until(&self) -> Result<MirrorPosition, StoreError> {
974 self.read(read::mirror_log_until).await
975 }
976}
977
978pub fn sizing_at(path: &Path, since: u64) -> Result<Option<sizing::History>, StoreError> {
983 if !path.exists() {
984 return Ok(None);
985 }
986 let read = |path: &str, flags| -> Result<sizing::History, StoreError> {
987 let db = Connection::open_with_flags(path, flags)?;
988 db.busy_timeout(Duration::from_secs(5))?;
989 sizing_of(&db, since)
990 };
991 let name = path.to_string_lossy();
992 match read(&name, OpenFlags::SQLITE_OPEN_READ_ONLY) {
993 Err(StoreError::Sqlite(error))
994 if matches!(
995 error.sqlite_error_code(),
996 Some(rusqlite::ErrorCode::ReadOnly | rusqlite::ErrorCode::CannotOpen)
997 ) =>
998 {
999 let mut wal = path.as_os_str().to_owned();
1001 wal.push("-wal");
1002 let wal = PathBuf::from(wal);
1003 if std::fs::metadata(&wal).is_ok_and(|wal| wal.len() > 0) {
1004 return Err(StoreError::Unread(wal));
1005 }
1006 let uri = format!(
1007 "file:{}?mode=ro&immutable=1",
1008 name.replace('%', "%25")
1009 .replace('?', "%3f")
1010 .replace('#', "%23")
1011 );
1012 read(
1013 &uri,
1014 OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_URI,
1015 )
1016 }
1017 history => history,
1018 }
1019 .map(Some)
1020}
1021
1022fn sizing_of(db: &Connection, since: u64) -> Result<sizing::History, StoreError> {
1024 match version(db)? {
1025 0 => return Ok(sizing::History::default()),
1026 VERSION => {}
1027 _ => read_as_current(db)?,
1028 }
1029 let mut history = read::sizing(db, since)?;
1030 let auto = read::auto_sizing(db)?;
1031 history.memory_set = auto.memory;
1032 history.slots_set = auto.slots;
1033 history.changes = read::sizing_changes(db, since.saturating_sub(sizing::memory::HISTORY))?;
1034 Ok(history)
1035}
1036
1037fn version(db: &Connection) -> Result<u32, StoreError> {
1039 let found: u32 = db.pragma_query_value(None, "user_version", |row| row.get(0))?;
1040 match found {
1041 0 | OLDEST..=VERSION => Ok(found),
1042 found if found > VERSION => Err(StoreError::Newer {
1043 found,
1044 supported: VERSION,
1045 }),
1046 found => Err(StoreError::Older {
1047 found,
1048 oldest: OLDEST,
1049 }),
1050 }
1051}
1052
1053fn read_as_current(db: &Connection) -> Result<(), StoreError> {
1057 let current = Connection::open_in_memory()?;
1058 current.execute_batch(SCHEMA)?;
1059 let tables: Vec<(String, String)> = current
1060 .prepare(
1061 "SELECT name, sql FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'",
1062 )?
1063 .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?
1064 .collect::<rusqlite::Result<_>>()?;
1065 let columns =
1066 |db: &Connection, table: &str| -> rusqlite::Result<Vec<(String, Option<String>)>> {
1067 db.prepare("SELECT name, dflt_value FROM pragma_table_info(?1, 'main')")?
1068 .query_map([table], |row| Ok((row.get(0)?, row.get(1)?)))?
1069 .collect()
1070 };
1071 for (table, sql) in tables {
1072 let found: Vec<String> = columns(db, &table)?
1073 .into_iter()
1074 .map(|(name, _)| name)
1075 .collect();
1076 if found.is_empty() {
1077 db.execute_batch(&sql.replacen("CREATE TABLE", "CREATE TEMP TABLE", 1))?;
1078 continue;
1079 }
1080 let added: Vec<String> = columns(¤t, &table)?
1081 .into_iter()
1082 .filter(|(name, _)| !found.contains(name))
1083 .map(|(name, default)| format!("{} AS {name}", default.as_deref().unwrap_or("NULL")))
1084 .collect();
1085 if !added.is_empty() {
1086 db.execute_batch(&format!(
1087 "CREATE TEMP VIEW {table} AS SELECT *, {} FROM main.{table}",
1088 added.join(", ")
1089 ))?;
1090 }
1091 }
1092 Ok(())
1093}
1094
1095fn configure(db: &Connection) -> rusqlite::Result<()> {
1097 db.pragma_update(None, "journal_mode", "wal")?;
1098 db.pragma_update(None, "synchronous", "normal")?;
1099 db.pragma_update(None, "foreign_keys", "on")?;
1100 db.busy_timeout(Duration::from_secs(5))
1101}
1102
1103fn prepare(db: &mut Connection) -> Result<(), StoreError> {
1106 let found = version(db)?;
1107 if found == 0 {
1108 let transaction = db.transaction()?;
1109 transaction.execute_batch(SCHEMA)?;
1110 transaction.pragma_update(None, "user_version", VERSION)?;
1111 transaction.commit()?;
1112 return Ok(());
1113 }
1114 for (version, migration) in MIGRATIONS.iter().filter(|(version, _)| *version > found) {
1115 let transaction = db.transaction()?;
1116 transaction.execute_batch(migration)?;
1117 transaction.pragma_update(None, "user_version", version)?;
1118 transaction.commit()?;
1119 }
1120 Ok(())
1121}
1122
1123#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1128struct Downtime {
1129 from: u64,
1130 until: u64,
1131}
1132
1133impl Downtime {
1134 fn before(db: &mut Connection, now: u64) -> Result<Option<Downtime>, StoreError> {
1138 let transaction = db.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?;
1141 let last: Option<u64> = transaction.query_row(
1142 "SELECT max(at) FROM (SELECT max(at) AS at FROM host_samples UNION ALL SELECT max(at) FROM events \
1143 UNION ALL SELECT value FROM meta WHERE key = 'opened_at')",
1144 [],
1145 |row| row.get(0),
1146 )?;
1147 let downtime = last
1148 .filter(|last| *last < now)
1149 .map(|from| Downtime { from, until: now });
1150 if let Some(downtime) = downtime {
1151 let queued: Vec<(u64, u64)> = transaction
1152 .prepare("SELECT id, queued_at FROM jobs WHERE state = 'queued'")?
1153 .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?
1154 .collect::<rusqlite::Result<_>>()?;
1155 for (job, queued_at) in queued {
1156 transaction.execute(
1157 "UPDATE jobs SET unseen = unseen + ?2 WHERE id = ?1",
1158 rusqlite::params![job, downtime.of(queued_at)],
1159 )?;
1160 }
1161 }
1162 transaction.execute(
1163 "INSERT OR REPLACE INTO meta (key, value) VALUES ('opened_at', ?1)",
1164 [now],
1165 )?;
1166 transaction.commit()?;
1167 Ok(downtime)
1168 }
1169
1170 fn of(self, queued_at: u64) -> u64 {
1172 self.until.saturating_sub(queued_at.max(self.from))
1173 }
1174
1175 fn after(self, queued_at: u64, started_at: u64) -> u64 {
1178 self.of(queued_at)
1179 .saturating_sub(started_at.saturating_sub(queued_at.max(self.from)))
1180 }
1181}
1182
1183fn logged<T>(what: &str, result: rusqlite::Result<T>) -> Option<T> {
1185 result
1186 .map_err(|error| warn!("job history: cannot {what}: {error}"))
1187 .ok()
1188}
1189
1190#[cfg(test)]
1191mod tests;