1use std::collections::BTreeMap;
8use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt};
9use std::path::{Path, PathBuf};
10use std::sync::atomic::{AtomicBool, Ordering};
11use std::sync::{Arc, Mutex, TryLockError};
12use std::time::Duration;
13
14use rusqlite::{Connection, OpenFlags};
15use tokio::sync::{OwnedSemaphorePermit, Semaphore, broadcast, mpsc, oneshot};
16use tracing::warn;
17
18use crate::api::{self, Event, EventEnvelope};
19use crate::config::History;
20use crate::github::{JobDetails, QueuedJob};
21use crate::observe::metrics::{CpuHeld, Events};
22use crate::scheduler::warm;
23use crate::sizing;
24use crate::sync::lock;
25
26mod prune;
27mod read;
28mod write;
29
30const SCHEMA: &str = include_str!("store/schema.sql");
32
33const OLDEST: u32 = 15;
35
36const MIGRATIONS: &[(u32, &str)] = &[
39 (
40 16,
41 "ALTER TABLE jobs ADD COLUMN unseen INTEGER NOT NULL DEFAULT 0;",
42 ),
43 (
44 17,
45 "ALTER TABLE jobs ADD COLUMN cpu_periods INTEGER;\n\
46 ALTER TABLE jobs ADD COLUMN cpu_throttled INTEGER;\n\
47 ALTER TABLE jobs ADD COLUMN cpu_stalled INTEGER;\n\
48 ALTER TABLE samples ADD COLUMN cpu_periods INTEGER;\n\
49 ALTER TABLE samples ADD COLUMN cpu_throttled INTEGER;\n\
50 ALTER TABLE samples ADD COLUMN cpu_stalled INTEGER;",
51 ),
52];
53
54const VERSION: u32 = match MIGRATIONS.last() {
56 Some((version, _)) => *version,
57 None => OLDEST,
58};
59
60pub(crate) const LOG_BACKLOG: u64 = 8 << 20;
63
64const HOST_SAMPLE_BATCH: usize = 30;
67
68const EVENT_BUFFER: usize = 4096;
70
71#[derive(Debug, thiserror::Error)]
73pub enum StoreError {
74 #[error("{0}")]
76 Sqlite(#[from] rusqlite::Error),
77 #[error(
79 "the job history database is version {found}, newer than the {supported} this runnerd reads; \
80 run a newer runnerd or move the database aside"
81 )]
82 Newer {
83 found: u32,
85 supported: u32,
87 },
88 #[error(
90 "the job history database is version {found}, from a runnerd before 0.0.19; this runnerd reads versions \
91 from {oldest} on: move the database aside"
92 )]
93 Older {
94 found: u32,
96 oldest: u32,
98 },
99 #[error("{0}")]
101 Io(#[from] std::io::Error),
102 #[error("a database read stopped: {0}")]
104 Join(#[from] tokio::task::JoinError),
105 #[error(
108 "{} holds changes not yet in the job history, which this user cannot read; run runnerd doctor as the user \
109 the daemon runs as",
110 .0.display()
111 )]
112 Unread(PathBuf),
113}
114
115#[derive(Debug, Clone, PartialEq, Eq)]
117pub struct RunnerRecord {
118 pub name: String,
120 pub repository: String,
122 pub profile: String,
124 pub slot: usize,
126 pub container: String,
128 pub started_at: u64,
130 pub memory_cap: Option<u64>,
132 pub cpu_cap: Option<u64>,
134 pub admission: Option<api::Admission>,
136}
137
138#[derive(Debug, Clone, PartialEq, Eq)]
140pub struct SampleRecord {
141 pub runner: String,
143 pub at: u64,
145 pub cpu: u64,
147 pub cpu_usage: u64,
149 pub memory: u64,
151 pub network_received: u64,
153 pub network_sent: u64,
155 pub events: Events,
157 pub cpu_held: CpuHeld,
159}
160
161#[derive(Debug, Clone, Default, PartialEq, Eq)]
163pub struct Measured {
164 pub at: Option<u64>,
166 pub bytes: BTreeMap<String, u64>,
168}
169
170#[derive(Debug, Clone, Default, PartialEq, Eq)]
173pub struct Announced {
174 pub profiles: Vec<api::ProfileStatus>,
176 pub repositories: Vec<api::RepositoryStatus>,
178}
179
180#[derive(Debug, Clone, PartialEq, Eq)]
182pub struct OpenJob {
183 pub id: u64,
185 pub repository: String,
187 pub state: api::JobState,
189 pub runner: Option<String>,
191 pub runner_removed_at: Option<u64>,
193 pub run_id: u64,
195 pub run_attempt: u32,
197 pub workflow: Option<String>,
199 pub queued_at: u64,
201 pub labels: Vec<String>,
203}
204
205#[derive(Debug, Clone, PartialEq, Eq)]
207pub struct SlotJob {
208 pub profile: String,
210 pub slot: usize,
212 pub key: crate::admission::Key,
214}
215
216#[derive(Debug, Clone, Copy, PartialEq, Eq)]
218pub struct LogPosition {
219 pub page: u64,
221 pub offset: u64,
223}
224
225#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
229pub struct MirrorPosition {
230 pub at: u64,
232 pub seen: u64,
234}
235
236#[derive(Debug)]
238enum Write {
239 Queued {
240 repository: String,
241 job: QueuedJob,
242 },
243 RunnerStarted(RunnerRecord),
244 RunnerAdmission {
245 runner: String,
246 admission: api::Admission,
247 },
248 RunnerOomKilled(String),
249 HostWaited {
250 jobs: Vec<u64>,
251 ms: u64,
252 },
253 BuildCache {
254 runner: String,
255 counts: api::BuildCacheCounts,
256 },
257 RunnerRemoved {
258 runner: String,
259 at: u64,
260 },
261 RunnersGone {
262 running: Vec<String>,
263 at: u64,
264 },
265 Claimed {
266 job: u64,
267 run_id: u64,
268 runner: String,
269 at: u64,
270 },
271 Details {
272 details: JobDetails,
273 },
274 Elsewhere {
275 details: JobDetails,
276 },
277 Lost {
278 job: u64,
279 at: u64,
280 },
281 Log {
282 job: u64,
283 lines: Vec<String>,
284 position: Option<LogPosition>,
286 max_bytes: u64,
287 _backlog: OwnedSemaphorePermit,
289 },
290 Samples(Vec<SampleRecord>),
291 HostSample(api::HostSample),
292 MirrorBytes {
293 bytes: Vec<(String, u64)>,
294 until: MirrorPosition,
295 done: oneshot::Sender<bool>,
297 },
298 CacheSizes {
299 at: u64,
300 sizes: BTreeMap<String, u64>,
301 },
302 Event(Box<Event>),
304 RecordMade(api::Record),
305 RecordSkipped {
306 job: u64,
307 why: String,
308 },
309 RecordRemoved {
310 job: u64,
311 at: u64,
312 by: api::RecordRemoval,
313 },
314 Paused {
315 repositories: Vec<String>,
316 done: oneshot::Sender<bool>,
318 },
319 Visibility {
320 repository: String,
321 visibility: api::Visibility,
322 at: u64,
323 },
324 SizingIntent {
325 change: api::SizingChange,
326 done: oneshot::Sender<Option<i64>>,
328 },
329 SizingResolved {
330 id: i64,
331 resolution: Resolution,
332 done: oneshot::Sender<bool>,
334 },
335 Notify {
336 change: NotifyChange,
337 done: oneshot::Sender<bool>,
339 },
340 Flush(oneshot::Sender<()>),
341}
342
343#[derive(Debug)]
345enum NotifyChange {
346 Record {
348 target: String,
349 set: Vec<(String, u64)>,
350 remove: Vec<String>,
351 },
352 Keep(Vec<String>),
354}
355
356#[derive(Debug, Clone, Copy, PartialEq, Eq)]
358pub enum Resolution {
359 Done,
361 Failed,
363 Uncertain,
366}
367
368#[derive(Debug, Clone, Default, PartialEq, Eq)]
370pub struct AutoSizing {
371 pub last: BTreeMap<String, u64>,
373 pub pending: Vec<(i64, api::SizingChange)>,
375 pub cpus: Vec<sizing::CpusSet>,
378 pub memory: Vec<sizing::MemorySet>,
380}
381
382#[derive(Debug, Clone)]
384pub struct Store {
385 writes: mpsc::UnboundedSender<Write>,
386 reader: Arc<Mutex<Connection>>,
387 pruner: Arc<Mutex<Connection>>,
389 closed: Arc<AtomicBool>,
391 held: Arc<Mutex<Vec<api::HostSample>>>,
394 latest_host: Arc<Mutex<Option<api::HostSample>>>,
396 latest_host_memory: Arc<Mutex<Option<api::HostMemory>>>,
398 latest_samples: Arc<Mutex<BTreeMap<String, SampleRecord>>>,
400 events: broadcast::Sender<Arc<EventEnvelope>>,
401 backlog: Arc<Semaphore>,
402}
403
404impl Store {
405 pub fn open(path: &Path) -> Result<Store, StoreError> {
410 Store::open_at(path, crate::time::now_ms())
411 }
412
413 fn open_at(path: &Path, now: u64) -> Result<Store, StoreError> {
415 if let Some(dir) = path.parent().filter(|dir| !dir.as_os_str().is_empty()) {
416 std::fs::DirBuilder::new()
417 .recursive(true)
418 .mode(0o700)
419 .create(dir)?;
420 }
421 std::fs::OpenOptions::new()
423 .create(true)
424 .append(true)
425 .mode(0o600)
426 .open(path)?;
427 let mut db = Connection::open(path)?;
428 configure(&db)?;
429 prepare(&mut db)?;
430 let downtime = Downtime::before(&mut db, now)?;
431 let reader = Connection::open_with_flags(
432 path,
433 OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX,
434 )?;
435 configure(&reader)?;
436 let pruner = Connection::open(path)?;
437 configure(&pruner)?;
438 let (events, _) = broadcast::channel(EVENT_BUFFER);
439 let (writes, receiver) = mpsc::unbounded_channel();
440 let thread_events = events.clone();
441 let held = Arc::new(Mutex::new(Vec::new()));
442 let thread_held = held.clone();
443 std::thread::Builder::new()
444 .name("runnerd-store".to_owned())
445 .spawn(move || write::run(db, downtime, receiver, &thread_events, &thread_held))?;
446 Ok(Store {
447 writes,
448 reader: Arc::new(Mutex::new(reader)),
449 pruner: Arc::new(Mutex::new(pruner)),
450 closed: Arc::new(AtomicBool::new(false)),
451 held,
452 latest_host: Arc::new(Mutex::new(None)),
453 latest_host_memory: Arc::new(Mutex::new(None)),
454 latest_samples: Arc::new(Mutex::new(BTreeMap::new())),
455 events,
456 backlog: Arc::new(Semaphore::new(
457 usize::try_from(LOG_BACKLOG).unwrap_or(usize::MAX),
458 )),
459 })
460 }
461
462 fn send(&self, write: Write) {
463 let _ = self.writes.send(write);
465 }
466
467 pub fn queued(&self, repository: &str, job: &QueuedJob) {
469 self.send(Write::Queued {
470 repository: repository.to_owned(),
471 job: job.clone(),
472 });
473 }
474
475 pub fn runner_started(&self, runner: RunnerRecord) {
477 self.send(Write::RunnerStarted(runner));
478 }
479
480 pub fn runner_removed(&self, runner: &str, at: u64) {
482 lock(&self.latest_samples).remove(runner);
483 self.send(Write::RunnerRemoved {
484 runner: runner.to_owned(),
485 at,
486 });
487 }
488
489 pub fn runners_gone(&self, running: Vec<String>, at: u64) {
492 lock(&self.latest_samples).retain(|runner, _| running.contains(runner));
493 self.send(Write::RunnersGone { running, at });
494 }
495
496 pub fn claimed(&self, job: u64, run_id: u64, runner: &str, at: u64) {
498 self.send(Write::Claimed {
499 job,
500 run_id,
501 runner: runner.to_owned(),
502 at,
503 });
504 }
505
506 pub fn details(&self, details: JobDetails) {
508 self.send(Write::Details { details });
509 }
510
511 pub fn elsewhere(&self, details: JobDetails) {
514 self.send(Write::Elsewhere { details });
515 }
516
517 pub fn lost(&self, job: u64, at: u64) {
519 self.send(Write::Lost { job, at });
520 }
521
522 pub async fn log(&self, job: u64, lines: Vec<String>, position: LogPosition, max_bytes: u64) {
525 let size = |line: &String| line.len() as u64 + 1;
526 let mut lines = lines.into_iter().map(|mut line| {
527 if size(&line) > LOG_BACKLOG {
529 let mut end = usize::try_from(LOG_BACKLOG / 2).unwrap_or(usize::MAX);
530 while !line.is_char_boundary(end) {
531 end -= 1;
532 }
533 line.truncate(end);
534 }
535 line
536 });
537 let mut next = lines.next();
538 loop {
539 let (mut part, mut bytes) = (Vec::new(), 0);
540 while let Some(line) = next.take_if(|line| bytes + size(line) <= LOG_BACKLOG) {
541 bytes += size(&line);
542 part.push(line);
543 next = lines.next();
544 }
545 let last = next.is_none();
546 let share = u32::try_from(bytes.max(1)).unwrap_or(u32::MAX);
547 let Ok(backlog) = self.backlog.clone().acquire_many_owned(share).await else {
549 return;
550 };
551 self.send(Write::Log {
552 job,
553 lines: part,
554 position: last.then_some(position),
555 max_bytes,
556 _backlog: backlog,
557 });
558 if last {
559 return;
560 }
561 }
562 }
563
564 pub fn samples(&self, samples: Vec<SampleRecord>) {
566 let mut latest = lock(&self.latest_samples);
567 for sample in &samples {
568 latest.insert(sample.runner.clone(), sample.clone());
569 }
570 drop(latest);
571 self.send(Write::Samples(samples));
572 }
573
574 pub fn host_sample(&self, sample: api::HostSample) {
577 *lock(&self.latest_host) = Some(sample.clone());
578 lock(&self.held).push(sample.clone());
579 self.send(Write::HostSample(sample));
580 }
581
582 pub fn latest_sample(&self, runner: &str) -> Option<SampleRecord> {
584 lock(&self.latest_samples).get(runner).cloned()
585 }
586
587 pub fn runner_admission(&self, runner: &str, admission: api::Admission) {
589 self.send(Write::RunnerAdmission {
590 runner: runner.to_owned(),
591 admission,
592 });
593 }
594
595 pub fn runner_oom_killed(&self, runner: &str) {
598 self.send(Write::RunnerOomKilled(runner.to_owned()));
599 }
600
601 pub fn build_cache(&self, runner: &str, counts: api::BuildCacheCounts) {
603 self.send(Write::BuildCache {
604 runner: runner.to_owned(),
605 counts,
606 });
607 }
608
609 pub fn host_memory(&self, memory: api::HostMemory) {
611 *lock(&self.latest_host_memory) = Some(memory);
612 }
613
614 pub fn latest_host_memory(&self) -> Option<api::HostMemory> {
616 lock(&self.latest_host_memory).clone()
617 }
618
619 pub fn host_waited(&self, jobs: Vec<u64>, ms: u64) {
622 self.send(Write::HostWaited { jobs, ms });
623 }
624
625 pub fn latest_host_sample(&self) -> Option<api::HostSample> {
627 lock(&self.latest_host).clone()
628 }
629
630 pub async fn mirror_bytes(&self, bytes: Vec<(String, u64)>, until: MirrorPosition) -> bool {
633 let (done, committed) = oneshot::channel();
634 self.send(Write::MirrorBytes { bytes, until, done });
635 committed.await.unwrap_or(false)
636 }
637
638 pub fn cache_sizes(&self, at: u64, sizes: BTreeMap<String, u64>) {
640 self.send(Write::CacheSizes { at, sizes });
641 }
642
643 pub fn event(&self, event: Event) {
645 self.send(Write::Event(Box::new(event)));
646 }
647
648 pub fn record_made(&self, record: api::Record) {
650 self.send(Write::RecordMade(record));
651 }
652
653 pub fn record_skipped(&self, job: u64, why: String) {
655 self.send(Write::RecordSkipped { job, why });
656 }
657
658 pub fn record_removed(&self, job: u64, at: u64, by: api::RecordRemoval) {
660 self.send(Write::RecordRemoved { job, at, by });
661 }
662
663 pub fn paused(&self, repositories: Vec<String>) -> impl Future<Output = bool> + use<> {
666 let (done, committed) = oneshot::channel();
667 self.send(Write::Paused { repositories, done });
668 async move { committed.await.unwrap_or(false) }
669 }
670
671 pub fn visibility(&self, repository: String, visibility: api::Visibility, at: u64) {
673 self.send(Write::Visibility {
674 repository,
675 visibility,
676 at,
677 });
678 }
679
680 pub fn sizing_intent(
683 &self,
684 change: api::SizingChange,
685 ) -> impl Future<Output = Option<i64>> + use<> {
686 let (done, committed) = oneshot::channel();
687 self.send(Write::SizingIntent { change, done });
688 async move { committed.await.ok().flatten() }
689 }
690
691 pub fn sizing_resolved(
694 &self,
695 id: i64,
696 resolution: Resolution,
697 ) -> impl Future<Output = bool> + use<> {
698 let (done, committed) = oneshot::channel();
699 self.send(Write::SizingResolved {
700 id,
701 resolution,
702 done,
703 });
704 async move { committed.await.unwrap_or(false) }
705 }
706
707 pub fn notify_record(
710 &self,
711 target: &str,
712 set: Vec<(String, u64)>,
713 remove: Vec<String>,
714 ) -> impl Future<Output = bool> + use<> {
715 let (done, committed) = oneshot::channel();
716 self.send(Write::Notify {
717 change: NotifyChange::Record {
718 target: target.to_owned(),
719 set,
720 remove,
721 },
722 done,
723 });
724 async move { committed.await.unwrap_or(false) }
725 }
726
727 pub fn notify_keep(&self, targets: Vec<String>) -> impl Future<Output = bool> + use<> {
729 let (done, committed) = oneshot::channel();
730 self.send(Write::Notify {
731 change: NotifyChange::Keep(targets),
732 done,
733 });
734 async move { committed.await.unwrap_or(false) }
735 }
736
737 pub fn prune(&self, now: u64, history: &History) -> tokio::task::JoinHandle<()> {
741 let pruner = self.pruner.clone();
742 let closed = self.closed.clone();
743 let history = history.clone();
744 tokio::task::spawn_blocking(move || {
745 let db = match pruner.try_lock() {
746 Ok(db) => db,
747 Err(TryLockError::Poisoned(poisoned)) => poisoned.into_inner(),
748 Err(TryLockError::WouldBlock) => return,
749 };
750 let db = prune::Db {
751 connection: &db,
752 stop: &closed,
753 };
754 if !db.stop.load(Ordering::Relaxed) {
755 prune::run(&db, now, &history);
756 }
757 })
758 }
759
760 pub fn close(&self) {
763 self.closed.store(true, Ordering::Relaxed);
764 }
765
766 pub async fn flush(&self) {
768 let (done, wait) = oneshot::channel();
769 self.send(Write::Flush(done));
770 let _ = wait.await;
771 }
772
773 pub fn subscribe(&self) -> broadcast::Receiver<Arc<EventEnvelope>> {
776 self.events.subscribe()
777 }
778
779 async fn read<T: Send + 'static>(
780 &self,
781 query: impl FnOnce(&Connection) -> rusqlite::Result<T> + Send + 'static,
782 ) -> Result<T, StoreError> {
783 let reader = self.reader.clone();
784 Ok(tokio::task::spawn_blocking(move || query(&lock(&reader))).await??)
786 }
787
788 pub async fn jobs(&self, query: api::JobQuery) -> Result<api::JobPage, StoreError> {
790 self.read(move |db| read::jobs(db, &query)).await
791 }
792
793 pub async fn job(&self, id: u64) -> Result<Option<api::Job>, StoreError> {
795 self.read(move |db| read::job(db, id)).await
796 }
797
798 pub async fn job_samples(
800 &self,
801 id: u64,
802 interval: Duration,
803 ) -> Result<Option<api::SampleSeries>, StoreError> {
804 self.read(move |db| read::job_samples(db, id, interval))
805 .await
806 }
807
808 pub async fn log_lines(
810 &self,
811 job: u64,
812 from: u64,
813 limit: u64,
814 ) -> Result<Vec<(u64, String)>, StoreError> {
815 self.read(move |db| read::log_lines(db, job, from, limit))
816 .await
817 }
818
819 pub async fn events_after(
821 &self,
822 after: u64,
823 limit: u64,
824 ) -> Result<Vec<EventEnvelope>, StoreError> {
825 self.read(move |db| read::events_after(db, after, limit))
826 .await
827 }
828
829 pub async fn sizing_changes(&self, since: u64) -> Result<Vec<api::SizingChange>, StoreError> {
831 self.read(move |db| read::sizing_changes(db, since)).await
832 }
833
834 pub async fn auto_sizing(&self) -> Result<AutoSizing, StoreError> {
836 self.read(read::auto_sizing).await
837 }
838
839 pub fn announced(&self) -> Result<Announced, StoreError> {
842 Ok(read::announced(&lock(&self.reader))?)
843 }
844
845 pub fn paused_repositories(&self) -> Result<Vec<String>, StoreError> {
847 Ok(read::paused(&lock(&self.reader))?)
848 }
849
850 pub async fn host_samples(&self, since: u64) -> Result<Vec<api::HostSample>, StoreError> {
852 let held = lock(&self.held).clone();
855 let mut samples = self.read(move |db| read::host_samples(db, since)).await?;
856 let stored = samples.last().map_or(0, |sample| sample.at);
857 samples.extend(
858 held.into_iter()
859 .filter(|sample| sample.at >= since && sample.at > stored),
860 );
861 Ok(samples)
862 }
863
864 pub async fn activity(&self, since: u64, now: u64) -> Result<api::Activity, StoreError> {
866 self.read(move |db| read::activity(db, since, now)).await
867 }
868
869 pub async fn stats(&self, query: api::StatsQuery, now: u64) -> Result<api::Stats, StoreError> {
871 self.read(move |db| read::stats(db, &query, now)).await
872 }
873
874 pub async fn sizing(&self, since: u64) -> Result<sizing::History, StoreError> {
876 self.read(move |db| read::sizing(db, since)).await
877 }
878
879 pub async fn pricing(&self, since: u64) -> Result<api::PricingHistory, StoreError> {
882 self.read(move |db| read::pricing(db, since)).await
883 }
884
885 pub async fn insights(&self, since: u64) -> Result<crate::insights::History, StoreError> {
888 self.read(move |db| read::insights(db, since)).await
889 }
890
891 pub async fn last_event(&self) -> Result<u64, StoreError> {
893 self.read(read::last_event).await
894 }
895
896 pub async fn notify_state(
898 &self,
899 ) -> Result<BTreeMap<String, BTreeMap<String, u64>>, StoreError> {
900 self.read(read::notify_state).await
901 }
902
903 pub async fn warm(&self, since: u64) -> Result<Vec<warm::Seen>, StoreError> {
905 self.read(move |db| read::warm(db, since)).await
906 }
907
908 pub async fn cache_sizes_now(&self) -> Result<Measured, StoreError> {
910 self.read(read::latest_cache_sizes).await
911 }
912
913 pub async fn build_cache_since(&self, since: u64) -> Result<api::BuildCacheCounts, StoreError> {
915 self.read(move |db| read::build_cache_since(db, since))
916 .await
917 }
918
919 pub async fn cache_history(&self, since: u64) -> Result<Vec<api::CacheSize>, StoreError> {
921 self.read(move |db| read::cache_history(db, since)).await
922 }
923
924 pub async fn open_jobs(&self) -> Result<Vec<OpenJob>, StoreError> {
926 self.read(read::open_jobs).await
927 }
928
929 pub async fn slot_jobs(&self) -> Result<Vec<SlotJob>, StoreError> {
931 self.read(read::slot_jobs).await
932 }
933
934 pub async fn runner_job(&self, runner: &str) -> Result<Option<u64>, StoreError> {
936 let runner = runner.to_owned();
937 self.read(move |db| read::runner_job(db, &runner)).await
938 }
939
940 pub async fn runner_admitted(
942 &self,
943 runner: &str,
944 ) -> Result<Option<api::Admission>, StoreError> {
945 let runner = runner.to_owned();
946 self.read(move |db| read::runner_admission(db, &runner))
947 .await
948 }
949
950 pub async fn log_position(&self, job: u64) -> Result<LogPosition, StoreError> {
952 self.read(move |db| read::log_position(db, job)).await
953 }
954
955 pub async fn mirror_log_until(&self) -> Result<MirrorPosition, StoreError> {
957 self.read(read::mirror_log_until).await
958 }
959}
960
961pub fn sizing_at(path: &Path, since: u64) -> Result<Option<sizing::History>, StoreError> {
966 if !path.exists() {
967 return Ok(None);
968 }
969 let read = |path: &str, flags| -> Result<sizing::History, StoreError> {
970 let db = Connection::open_with_flags(path, flags)?;
971 db.busy_timeout(Duration::from_secs(5))?;
972 sizing_of(&db, since)
973 };
974 let name = path.to_string_lossy();
975 match read(&name, OpenFlags::SQLITE_OPEN_READ_ONLY) {
976 Err(StoreError::Sqlite(error))
977 if matches!(
978 error.sqlite_error_code(),
979 Some(rusqlite::ErrorCode::ReadOnly | rusqlite::ErrorCode::CannotOpen)
980 ) =>
981 {
982 let mut wal = path.as_os_str().to_owned();
984 wal.push("-wal");
985 let wal = PathBuf::from(wal);
986 if std::fs::metadata(&wal).is_ok_and(|wal| wal.len() > 0) {
987 return Err(StoreError::Unread(wal));
988 }
989 let uri = format!(
990 "file:{}?mode=ro&immutable=1",
991 name.replace('%', "%25")
992 .replace('?', "%3f")
993 .replace('#', "%23")
994 );
995 read(
996 &uri,
997 OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_URI,
998 )
999 }
1000 history => history,
1001 }
1002 .map(Some)
1003}
1004
1005fn sizing_of(db: &Connection, since: u64) -> Result<sizing::History, StoreError> {
1007 match version(db)? {
1008 0 => return Ok(sizing::History::default()),
1009 VERSION => {}
1010 _ => read_as_current(db)?,
1011 }
1012 let mut history = read::sizing(db, since)?;
1013 history.memory_set = read::auto_sizing(db)?.memory;
1014 history.changes = read::sizing_changes(db, since.saturating_sub(sizing::memory::HISTORY))?;
1015 Ok(history)
1016}
1017
1018fn version(db: &Connection) -> Result<u32, StoreError> {
1020 let found: u32 = db.pragma_query_value(None, "user_version", |row| row.get(0))?;
1021 match found {
1022 0 | OLDEST..=VERSION => Ok(found),
1023 found if found > VERSION => Err(StoreError::Newer {
1024 found,
1025 supported: VERSION,
1026 }),
1027 found => Err(StoreError::Older {
1028 found,
1029 oldest: OLDEST,
1030 }),
1031 }
1032}
1033
1034fn read_as_current(db: &Connection) -> Result<(), StoreError> {
1038 let current = Connection::open_in_memory()?;
1039 current.execute_batch(SCHEMA)?;
1040 let tables: Vec<(String, String)> = current
1041 .prepare(
1042 "SELECT name, sql FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'",
1043 )?
1044 .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?
1045 .collect::<rusqlite::Result<_>>()?;
1046 let columns =
1047 |db: &Connection, table: &str| -> rusqlite::Result<Vec<(String, Option<String>)>> {
1048 db.prepare("SELECT name, dflt_value FROM pragma_table_info(?1, 'main')")?
1049 .query_map([table], |row| Ok((row.get(0)?, row.get(1)?)))?
1050 .collect()
1051 };
1052 for (table, sql) in tables {
1053 let found: Vec<String> = columns(db, &table)?
1054 .into_iter()
1055 .map(|(name, _)| name)
1056 .collect();
1057 if found.is_empty() {
1058 db.execute_batch(&sql.replacen("CREATE TABLE", "CREATE TEMP TABLE", 1))?;
1059 continue;
1060 }
1061 let added: Vec<String> = columns(¤t, &table)?
1062 .into_iter()
1063 .filter(|(name, _)| !found.contains(name))
1064 .map(|(name, default)| format!("{} AS {name}", default.as_deref().unwrap_or("NULL")))
1065 .collect();
1066 if !added.is_empty() {
1067 db.execute_batch(&format!(
1068 "CREATE TEMP VIEW {table} AS SELECT *, {} FROM main.{table}",
1069 added.join(", ")
1070 ))?;
1071 }
1072 }
1073 Ok(())
1074}
1075
1076fn configure(db: &Connection) -> rusqlite::Result<()> {
1078 db.pragma_update(None, "journal_mode", "wal")?;
1079 db.pragma_update(None, "synchronous", "normal")?;
1080 db.pragma_update(None, "foreign_keys", "on")?;
1081 db.busy_timeout(Duration::from_secs(5))
1082}
1083
1084fn prepare(db: &mut Connection) -> Result<(), StoreError> {
1087 let found = version(db)?;
1088 if found == 0 {
1089 let transaction = db.transaction()?;
1090 transaction.execute_batch(SCHEMA)?;
1091 transaction.pragma_update(None, "user_version", VERSION)?;
1092 transaction.commit()?;
1093 return Ok(());
1094 }
1095 for (version, migration) in MIGRATIONS.iter().filter(|(version, _)| *version > found) {
1096 let transaction = db.transaction()?;
1097 transaction.execute_batch(migration)?;
1098 transaction.pragma_update(None, "user_version", version)?;
1099 transaction.commit()?;
1100 }
1101 Ok(())
1102}
1103
1104#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1109struct Downtime {
1110 from: u64,
1111 until: u64,
1112}
1113
1114impl Downtime {
1115 fn before(db: &mut Connection, now: u64) -> Result<Option<Downtime>, StoreError> {
1119 let transaction = db.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?;
1122 let last: Option<u64> = transaction.query_row(
1123 "SELECT max(at) FROM (SELECT max(at) AS at FROM host_samples UNION ALL SELECT max(at) FROM events \
1124 UNION ALL SELECT value FROM meta WHERE key = 'opened_at')",
1125 [],
1126 |row| row.get(0),
1127 )?;
1128 let downtime = last
1129 .filter(|last| *last < now)
1130 .map(|from| Downtime { from, until: now });
1131 if let Some(downtime) = downtime {
1132 let queued: Vec<(u64, u64)> = transaction
1133 .prepare("SELECT id, queued_at FROM jobs WHERE state = 'queued'")?
1134 .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?
1135 .collect::<rusqlite::Result<_>>()?;
1136 for (job, queued_at) in queued {
1137 transaction.execute(
1138 "UPDATE jobs SET unseen = unseen + ?2 WHERE id = ?1",
1139 rusqlite::params![job, downtime.of(queued_at)],
1140 )?;
1141 }
1142 }
1143 transaction.execute(
1144 "INSERT OR REPLACE INTO meta (key, value) VALUES ('opened_at', ?1)",
1145 [now],
1146 )?;
1147 transaction.commit()?;
1148 Ok(downtime)
1149 }
1150
1151 fn of(self, queued_at: u64) -> u64 {
1153 self.until.saturating_sub(queued_at.max(self.from))
1154 }
1155
1156 fn after(self, queued_at: u64, started_at: u64) -> u64 {
1159 self.of(queued_at)
1160 .saturating_sub(started_at.saturating_sub(queued_at.max(self.from)))
1161 }
1162}
1163
1164fn logged<T>(what: &str, result: rusqlite::Result<T>) -> Option<T> {
1166 result
1167 .map_err(|error| warn!("job history: cannot {what}: {error}"))
1168 .ok()
1169}
1170
1171#[cfg(test)]
1172mod tests;