1use std::collections::{BTreeMap, BTreeSet};
8use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt};
9use std::path::{Path, PathBuf};
10use std::sync::atomic::{AtomicBool, Ordering};
11use std::sync::{Arc, Mutex, TryLockError};
12use std::time::Duration;
13
14use rusqlite::{Connection, OpenFlags};
15use tokio::sync::{OwnedSemaphorePermit, Semaphore, broadcast, mpsc, oneshot};
16use tracing::warn;
17
18use crate::admission::trace::Trace;
19use crate::api::{self, Event, EventEnvelope};
20use crate::config::History;
21use crate::github::{JobDetails, NamedRun, QueuedJob};
22use crate::observe::metrics::{CpuHeld, Events};
23use crate::scheduler::warm;
24use crate::sizing;
25use crate::sync::lock;
26
27mod lens;
28mod outline;
29mod prune;
30mod read;
31mod run;
32mod write;
33
34pub use run::RunJobs;
35
36const SCHEMA: &str = include_str!("store/schema.sql");
38
39const OLDEST: u32 = 15;
41
42const MIGRATIONS: &[(u32, &str)] = &[
45 (
46 16,
47 "ALTER TABLE jobs ADD COLUMN unseen INTEGER NOT NULL DEFAULT 0;",
48 ),
49 (
50 17,
51 "ALTER TABLE jobs ADD COLUMN cpu_periods INTEGER;\n\
52 ALTER TABLE jobs ADD COLUMN cpu_throttled INTEGER;\n\
53 ALTER TABLE jobs ADD COLUMN cpu_stalled INTEGER;\n\
54 ALTER TABLE samples ADD COLUMN cpu_periods INTEGER;\n\
55 ALTER TABLE samples ADD COLUMN cpu_throttled INTEGER;\n\
56 ALTER TABLE samples ADD COLUMN cpu_stalled INTEGER;",
57 ),
58 (
59 18,
60 "ALTER TABLE jobs ADD COLUMN slot_waited INTEGER NOT NULL DEFAULT 0;",
61 ),
62 (
63 19,
64 "ALTER TABLE jobs ADD COLUMN seen_at INTEGER;\n\
65 ALTER TABLE runners ADD COLUMN purpose TEXT;\n\
66 ALTER TABLE runners ADD COLUMN stopped_for TEXT;",
67 ),
68 (
69 20,
70 "ALTER TABLE jobs ADD COLUMN runner_fault INTEGER NOT NULL DEFAULT 0;",
71 ),
72 (21, "ALTER TABLE runners ADD COLUMN cancel_by INTEGER;"),
73 (
74 22,
75 "CREATE TABLE log_outlines (job INTEGER PRIMARY KEY NOT NULL, from_line INTEGER NOT NULL, \
76 errors INTEGER NOT NULL, warnings INTEGER NOT NULL, groups_kept INTEGER NOT NULL, \
77 marks_kept INTEGER NOT NULL, sections_kept INTEGER NOT NULL, dropped INTEGER NOT NULL, \
78 hidden INTEGER NOT NULL);\n\
79 CREATE TABLE log_groups (job INTEGER NOT NULL, start INTEGER NOT NULL, end_line INTEGER, \
80 depth INTEGER NOT NULL, title TEXT NOT NULL, PRIMARY KEY (job, start)) WITHOUT ROWID;\n\
81 CREATE INDEX log_groups_open ON log_groups (job, start) WHERE end_line IS NULL;\n\
82 CREATE TABLE log_marks (job INTEGER NOT NULL, line INTEGER NOT NULL, kind TEXT NOT NULL, \
83 PRIMARY KEY (job, line)) WITHOUT ROWID;",
84 ),
85 (
86 23,
87 "CREATE TABLE sessions (id INTEGER PRIMARY KEY, started_at INTEGER NOT NULL, sampled_at INTEGER);\n\
88 ALTER TABLE records ADD COLUMN gone_at INTEGER;",
89 ),
90];
91
92const VERSION: u32 = match MIGRATIONS.last() {
94 Some((version, _)) => *version,
95 None => OLDEST,
96};
97
98pub(crate) const LOG_BACKLOG: u64 = 8 << 20;
101
102const HOST_SAMPLE_BATCH: usize = 30;
105
106const EVENT_BUFFER: usize = 4096;
108
109#[derive(Debug, thiserror::Error)]
111pub enum StoreError {
112 #[error("{0}")]
114 Sqlite(#[from] rusqlite::Error),
115 #[error(
117 "the job history database is version {found}, newer than the {supported} this runnerd reads; \
118 run a newer runnerd or move the database aside"
119 )]
120 Newer {
121 found: u32,
123 supported: u32,
125 },
126 #[error(
128 "the job history database is version {found}, from a runnerd before 0.0.19; this runnerd reads versions \
129 from {oldest} on: move the database aside"
130 )]
131 Older {
132 found: u32,
134 oldest: u32,
136 },
137 #[error("{0}")]
139 Io(#[from] std::io::Error),
140 #[error("a database read stopped: {0}")]
142 Join(#[from] tokio::task::JoinError),
143 #[error(
146 "{} holds changes not yet in the job history, which this user cannot read; run runnerd doctor as the user \
147 the daemon runs as",
148 .0.display()
149 )]
150 Unread(PathBuf),
151}
152
153impl StoreError {
154 pub fn is_shutdown(&self) -> bool {
157 matches!(self, StoreError::Join(join) if join.is_cancelled())
158 }
159}
160
161#[derive(Debug, Clone, PartialEq, Eq)]
163pub struct RunnerRecord {
164 pub name: String,
166 pub repository: String,
168 pub profile: String,
170 pub slot: usize,
172 pub container: String,
174 pub started_at: u64,
176 pub memory_cap: Option<u64>,
178 pub cpu_cap: Option<u64>,
180 pub admission: Option<api::Admission>,
182 pub purpose: Option<&'static str>,
184}
185
186#[derive(Debug, Clone, PartialEq, Eq)]
188pub struct SampleRecord {
189 pub runner: String,
191 pub at: u64,
193 pub cpu: u64,
195 pub cpu_usage: u64,
197 pub memory: u64,
199 pub network_received: u64,
201 pub network_sent: u64,
203 pub events: Events,
205 pub cpu_held: CpuHeld,
207}
208
209#[derive(Debug, Clone, Default, PartialEq, Eq)]
211pub struct Measured {
212 pub at: Option<u64>,
214 pub bytes: BTreeMap<String, u64>,
216}
217
218#[derive(Debug, Clone, Default, PartialEq, Eq)]
221pub struct Announced {
222 pub profiles: Vec<api::ProfileStatus>,
224 pub repositories: Vec<api::RepositoryStatus>,
226}
227
228#[derive(Debug, Clone, PartialEq, Eq)]
230pub struct OpenJob {
231 pub id: u64,
233 pub repository: String,
235 pub state: api::JobState,
237 pub runner: Option<String>,
239 pub runner_removed_at: Option<u64>,
241 pub run_id: u64,
243 pub run_attempt: u32,
245 pub workflow: Option<String>,
247 pub queued_at: u64,
249 pub labels: Vec<String>,
251}
252
253#[derive(Debug, Clone, PartialEq, Eq)]
255pub struct SlotJob {
256 pub profile: String,
258 pub slot: usize,
260 pub key: crate::admission::Key,
262}
263
264#[derive(Debug, Clone, Copy, PartialEq, Eq)]
266pub struct LogPosition {
267 pub page: u64,
269 pub offset: u64,
271}
272
273#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
277pub struct MirrorPosition {
278 pub at: u64,
280 pub seen: u64,
282}
283
284#[derive(Debug)]
286enum Write {
287 Queued {
288 repository: String,
289 job: QueuedJob,
290 seen_at: u64,
291 },
292 RunnerStarted(RunnerRecord),
293 RunnerAdmission {
294 runner: String,
295 admission: api::Admission,
296 },
297 RunnerOomKilled(String),
298 RunnerStoppedFor {
299 runner: String,
300 reason: &'static str,
301 },
302 RunnerCancelBy {
303 runner: String,
304 at: u64,
305 },
306 HostWaited {
307 jobs: Vec<u64>,
308 ms: u64,
309 },
310 SlotWaited {
311 jobs: Vec<u64>,
312 ms: u64,
313 },
314 BuildCache {
315 runner: String,
316 counts: api::BuildCacheCounts,
317 },
318 RunnerRemoved {
319 runner: String,
320 at: u64,
321 },
322 RunnersGone {
323 running: Vec<String>,
324 at: u64,
325 },
326 Claimed {
327 job: u64,
328 run_id: u64,
329 runner: String,
330 at: u64,
331 },
332 Details {
333 details: JobDetails,
334 },
335 WorkflowNamed(NamedRun),
336 Elsewhere {
337 details: JobDetails,
338 },
339 Lost {
340 job: u64,
341 at: u64,
342 },
343 Unanswered {
344 job: u64,
345 at: u64,
346 },
347 Log {
348 job: u64,
349 lines: Vec<String>,
350 position: Option<LogPosition>,
352 max_bytes: u64,
353 _backlog: OwnedSemaphorePermit,
355 },
356 Samples(Vec<SampleRecord>),
357 HostSample(api::HostSample),
358 MirrorBytes {
359 bytes: Vec<(String, u64)>,
360 until: MirrorPosition,
361 done: oneshot::Sender<bool>,
363 },
364 CacheSizes {
365 at: u64,
366 sizes: BTreeMap<String, u64>,
367 },
368 Event(Box<Event>),
370 RecordMade(api::Record),
371 RecordSkipped {
372 job: u64,
373 why: String,
374 },
375 RecordRemoved {
376 job: u64,
377 at: u64,
378 by: api::RecordRemoval,
379 },
380 RecordsListed {
381 jobs: Vec<u64>,
382 at: u64,
383 },
384 Paused {
385 repositories: Vec<String>,
386 done: oneshot::Sender<bool>,
388 },
389 Visibility {
390 repository: String,
391 visibility: api::Visibility,
392 at: u64,
393 },
394 SizingIntent {
395 change: api::SizingChange,
396 done: oneshot::Sender<Option<i64>>,
398 },
399 SizingResolved {
400 id: i64,
401 resolution: Resolution,
402 done: oneshot::Sender<bool>,
404 },
405 Notify {
406 change: NotifyChange,
407 done: oneshot::Sender<bool>,
409 },
410 RunnerNamed {
411 secs: u64,
412 done: oneshot::Sender<bool>,
414 },
415 Flush(oneshot::Sender<()>),
416}
417
418#[derive(Debug)]
420enum NotifyChange {
421 Record {
423 target: String,
424 set: Vec<(String, u64)>,
425 remove: Vec<String>,
426 },
427 Keep(Vec<String>),
429}
430
431#[derive(Debug, Clone, Copy, PartialEq, Eq)]
433pub enum Resolution {
434 Done,
436 Failed,
438 Uncertain,
441}
442
443#[derive(Debug, Clone, Default, PartialEq, Eq)]
445pub struct AutoSizing {
446 pub last: BTreeMap<String, u64>,
448 pub pending: Vec<(i64, api::SizingChange)>,
450 pub cpus: Vec<sizing::CpusSet>,
453 pub memory: Vec<sizing::MemorySet>,
455 pub slots: Vec<sizing::SlotsSet>,
458}
459
460#[derive(Debug, Clone)]
462pub struct Store {
463 writes: mpsc::UnboundedSender<Write>,
464 reader: Arc<Mutex<Connection>>,
465 pruner: Arc<Mutex<Connection>>,
467 closed: Arc<AtomicBool>,
469 held: Arc<Mutex<Vec<api::HostSample>>>,
472 latest_host: Arc<Mutex<Option<api::HostSample>>>,
474 latest_host_memory: Arc<Mutex<Option<api::HostMemory>>>,
476 host_waiting: Arc<Mutex<(BTreeSet<u64>, u64)>>,
478 latest_samples: Arc<Mutex<BTreeMap<String, SampleRecord>>>,
480 gone: Arc<Mutex<BTreeSet<String>>>,
482 events: broadcast::Sender<Arc<EventEnvelope>>,
483 backlog: Arc<Semaphore>,
484}
485
486impl Store {
487 pub fn open(path: &Path) -> Result<Store, StoreError> {
492 Store::open_at(path, crate::time::now_ms())
493 }
494
495 fn open_at(path: &Path, now: u64) -> Result<Store, StoreError> {
497 if let Some(dir) = path.parent().filter(|dir| !dir.as_os_str().is_empty()) {
498 std::fs::DirBuilder::new()
499 .recursive(true)
500 .mode(0o700)
501 .create(dir)?;
502 }
503 std::fs::OpenOptions::new()
505 .create(true)
506 .append(true)
507 .mode(0o600)
508 .open(path)?;
509 let mut db = Connection::open(path)?;
510 configure(&db)?;
511 prepare(&mut db)?;
512 let downtime = Downtime::before(&mut db, now)?;
513 db.execute("INSERT INTO sessions (started_at) VALUES (?1)", [now])?;
514 let session = db.last_insert_rowid();
515 let reader = Connection::open_with_flags(
516 path,
517 OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX,
518 )?;
519 configure(&reader)?;
520 let pruner = Connection::open(path)?;
521 configure(&pruner)?;
522 let (events, _) = broadcast::channel(EVENT_BUFFER);
523 let (writes, receiver) = mpsc::unbounded_channel();
524 let thread_events = events.clone();
525 let held = Arc::new(Mutex::new(Vec::new()));
526 let thread_held = held.clone();
527 std::thread::Builder::new()
528 .name("runnerd-store".to_owned())
529 .spawn(move || {
530 write::run(
531 write::Started {
532 db,
533 downtime,
534 session,
535 },
536 receiver,
537 &thread_events,
538 &thread_held,
539 );
540 })?;
541 Ok(Store {
542 writes,
543 reader: Arc::new(Mutex::new(reader)),
544 pruner: Arc::new(Mutex::new(pruner)),
545 closed: Arc::new(AtomicBool::new(false)),
546 held,
547 latest_host: Arc::new(Mutex::new(None)),
548 latest_host_memory: Arc::new(Mutex::new(None)),
549 host_waiting: Arc::new(Mutex::new((BTreeSet::new(), 0))),
550 latest_samples: Arc::new(Mutex::new(BTreeMap::new())),
551 gone: Arc::new(Mutex::new(BTreeSet::new())),
552 events,
553 backlog: Arc::new(Semaphore::new(
554 usize::try_from(LOG_BACKLOG).unwrap_or(usize::MAX),
555 )),
556 })
557 }
558
559 fn send(&self, write: Write) {
560 let _ = self.writes.send(write);
562 }
563
564 pub fn queued(&self, repository: &str, job: &QueuedJob, seen_at: u64) {
568 self.send(Write::Queued {
569 repository: repository.to_owned(),
570 job: job.clone(),
571 seen_at,
572 });
573 }
574
575 pub fn runner_started(&self, runner: RunnerRecord) {
577 self.send(Write::RunnerStarted(runner));
578 }
579
580 pub fn runner_removed(&self, runner: &str, at: u64) {
582 lock(&self.latest_samples).remove(runner);
583 lock(&self.gone).remove(runner);
584 self.send(Write::RunnerRemoved {
585 runner: runner.to_owned(),
586 at,
587 });
588 }
589
590 pub fn runners_gone(&self, running: Vec<String>, at: u64) {
593 lock(&self.latest_samples).retain(|runner, _| running.contains(runner));
594 lock(&self.gone).retain(|runner| running.contains(runner));
595 self.send(Write::RunnersGone { running, at });
596 }
597
598 pub fn claimed(&self, job: u64, run_id: u64, runner: &str, at: u64) {
600 self.send(Write::Claimed {
601 job,
602 run_id,
603 runner: runner.to_owned(),
604 at,
605 });
606 }
607
608 pub fn details(&self, details: JobDetails) {
610 self.send(Write::Details { details });
611 }
612
613 pub fn workflow_named(&self, named: NamedRun) {
616 self.send(Write::WorkflowNamed(named));
617 }
618
619 pub fn elsewhere(&self, details: JobDetails) {
622 self.send(Write::Elsewhere { details });
623 }
624
625 pub fn unanswered(&self, job: u64, at: u64) {
628 self.send(Write::Unanswered { job, at });
629 }
630
631 pub fn lost(&self, job: u64, at: u64) {
633 self.send(Write::Lost { job, at });
634 }
635
636 pub async fn log(&self, job: u64, lines: Vec<String>, position: LogPosition, max_bytes: u64) {
639 let size = |line: &String| line.len() as u64 + 1;
640 let mut lines = lines.into_iter().map(|mut line| {
641 if size(&line) > LOG_BACKLOG {
643 let mut end = usize::try_from(LOG_BACKLOG / 2).unwrap_or(usize::MAX);
644 while !line.is_char_boundary(end) {
645 end -= 1;
646 }
647 line.truncate(end);
648 }
649 line
650 });
651 let mut next = lines.next();
652 loop {
653 let (mut part, mut bytes) = (Vec::new(), 0);
654 while let Some(line) = next.take_if(|line| bytes + size(line) <= LOG_BACKLOG) {
655 bytes += size(&line);
656 part.push(line);
657 next = lines.next();
658 }
659 let last = next.is_none();
660 let share = u32::try_from(bytes.max(1)).unwrap_or(u32::MAX);
661 let Ok(backlog) = self.backlog.clone().acquire_many_owned(share).await else {
663 return;
664 };
665 self.send(Write::Log {
666 job,
667 lines: part,
668 position: last.then_some(position),
669 max_bytes,
670 _backlog: backlog,
671 });
672 if last {
673 return;
674 }
675 }
676 }
677
678 pub fn samples(&self, samples: Vec<SampleRecord>) {
680 let mut latest = lock(&self.latest_samples);
681 for sample in &samples {
682 latest.insert(sample.runner.clone(), sample.clone());
683 }
684 drop(latest);
685 self.send(Write::Samples(samples));
686 }
687
688 pub fn host_sample(&self, sample: api::HostSample) {
691 *lock(&self.latest_host) = Some(sample.clone());
692 lock(&self.held).push(sample.clone());
693 self.send(Write::HostSample(sample));
694 }
695
696 pub fn latest_sample(&self, runner: &str) -> Option<SampleRecord> {
698 lock(&self.latest_samples).get(runner).cloned()
699 }
700
701 pub fn containers_gone(&self, runners: Vec<String>) {
704 lock(&self.gone).extend(runners);
705 }
706
707 pub fn containers_read(&self, runners: &[String]) {
710 let mut gone = lock(&self.gone);
711 for runner in runners {
712 gone.remove(runner);
713 }
714 }
715
716 pub fn container_gone(&self, runner: &str) -> bool {
718 lock(&self.gone).contains(runner)
719 }
720
721 pub fn runner_admission(&self, runner: &str, admission: api::Admission) {
723 self.send(Write::RunnerAdmission {
724 runner: runner.to_owned(),
725 admission,
726 });
727 }
728
729 pub fn runner_oom_killed(&self, runner: &str) {
732 self.send(Write::RunnerOomKilled(runner.to_owned()));
733 }
734
735 pub fn runner_cancel_by(&self, runner: &str, at: u64) {
738 self.send(Write::RunnerCancelBy {
739 runner: runner.to_owned(),
740 at,
741 });
742 }
743
744 pub fn runner_stopped_for(&self, runner: &str, reason: &'static str) {
746 self.send(Write::RunnerStoppedFor {
747 runner: runner.to_owned(),
748 reason,
749 });
750 }
751
752 pub fn build_cache(&self, runner: &str, counts: api::BuildCacheCounts) {
754 self.send(Write::BuildCache {
755 runner: runner.to_owned(),
756 counts,
757 });
758 }
759
760 pub fn host_memory(&self, memory: api::HostMemory) {
762 *lock(&self.latest_host_memory) = Some(memory);
763 }
764
765 pub fn latest_host_memory(&self) -> Option<api::HostMemory> {
767 lock(&self.latest_host_memory).clone()
768 }
769
770 pub fn host_waiting(&self, jobs: BTreeSet<u64>, at: u64) {
773 *lock(&self.host_waiting) = (jobs, at);
774 }
775
776 pub fn host_waited(&self, jobs: Vec<u64>, ms: u64) {
779 self.send(Write::HostWaited { jobs, ms });
780 }
781
782 pub fn slot_waited(&self, jobs: Vec<u64>, ms: u64) {
785 self.send(Write::SlotWaited { jobs, ms });
786 }
787
788 pub fn latest_host_sample(&self) -> Option<api::HostSample> {
790 lock(&self.latest_host).clone()
791 }
792
793 pub async fn mirror_bytes(&self, bytes: Vec<(String, u64)>, until: MirrorPosition) -> bool {
796 let (done, committed) = oneshot::channel();
797 self.send(Write::MirrorBytes { bytes, until, done });
798 committed.await.unwrap_or(false)
799 }
800
801 pub fn cache_sizes(&self, at: u64, sizes: BTreeMap<String, u64>) {
803 self.send(Write::CacheSizes { at, sizes });
804 }
805
806 pub fn event(&self, event: Event) {
808 self.send(Write::Event(Box::new(event)));
809 }
810
811 pub fn record_made(&self, record: api::Record) {
813 self.send(Write::RecordMade(record));
814 }
815
816 pub fn record_skipped(&self, job: u64, why: String) {
818 self.send(Write::RecordSkipped { job, why });
819 }
820
821 pub fn record_removed(&self, job: u64, at: u64, by: api::RecordRemoval) {
823 self.send(Write::RecordRemoved { job, at, by });
824 }
825
826 pub fn records_listed(&self, jobs: Vec<u64>, at: u64) {
830 self.send(Write::RecordsListed { jobs, at });
831 }
832
833 pub fn paused(&self, repositories: Vec<String>) -> impl Future<Output = bool> + use<> {
836 let (done, committed) = oneshot::channel();
837 self.send(Write::Paused { repositories, done });
838 async move { committed.await.unwrap_or(false) }
839 }
840
841 pub async fn runner_named(&self, secs: u64) -> bool {
845 let (done, committed) = oneshot::channel();
846 self.send(Write::RunnerNamed { secs, done });
847 tokio::task::spawn_blocking(move || committed.blocking_recv().unwrap_or(false))
850 .await
851 .unwrap_or(false)
852 }
853
854 pub fn visibility(&self, repository: String, visibility: api::Visibility, at: u64) {
856 self.send(Write::Visibility {
857 repository,
858 visibility,
859 at,
860 });
861 }
862
863 pub fn sizing_intent(
866 &self,
867 change: api::SizingChange,
868 ) -> impl Future<Output = Option<i64>> + use<> {
869 let (done, committed) = oneshot::channel();
870 self.send(Write::SizingIntent { change, done });
871 async move { committed.await.ok().flatten() }
872 }
873
874 pub fn sizing_resolved(
877 &self,
878 id: i64,
879 resolution: Resolution,
880 ) -> impl Future<Output = bool> + use<> {
881 let (done, committed) = oneshot::channel();
882 self.send(Write::SizingResolved {
883 id,
884 resolution,
885 done,
886 });
887 async move { committed.await.unwrap_or(false) }
888 }
889
890 pub fn notify_record(
893 &self,
894 target: &str,
895 set: Vec<(String, u64)>,
896 remove: Vec<String>,
897 ) -> impl Future<Output = bool> + use<> {
898 let (done, committed) = oneshot::channel();
899 self.send(Write::Notify {
900 change: NotifyChange::Record {
901 target: target.to_owned(),
902 set,
903 remove,
904 },
905 done,
906 });
907 async move { committed.await.unwrap_or(false) }
908 }
909
910 pub fn notify_keep(&self, targets: Vec<String>) -> impl Future<Output = bool> + use<> {
912 let (done, committed) = oneshot::channel();
913 self.send(Write::Notify {
914 change: NotifyChange::Keep(targets),
915 done,
916 });
917 async move { committed.await.unwrap_or(false) }
918 }
919
920 pub fn prune(&self, now: u64, history: &History) -> tokio::task::JoinHandle<()> {
924 let pruner = self.pruner.clone();
925 let closed = self.closed.clone();
926 let history = history.clone();
927 tokio::task::spawn_blocking(move || {
928 let db = match pruner.try_lock() {
929 Ok(db) => db,
930 Err(TryLockError::Poisoned(poisoned)) => poisoned.into_inner(),
931 Err(TryLockError::WouldBlock) => return,
932 };
933 let db = prune::Db {
934 connection: &db,
935 stop: &closed,
936 };
937 if !db.stop.load(Ordering::Relaxed) {
938 prune::run(&db, now, &history);
939 }
940 })
941 }
942
943 pub fn close(&self) {
946 self.closed.store(true, Ordering::Relaxed);
947 }
948
949 pub async fn flush(&self) {
951 let (done, wait) = oneshot::channel();
952 self.send(Write::Flush(done));
953 let _ = wait.await;
954 }
955
956 pub fn subscribe(&self) -> broadcast::Receiver<Arc<EventEnvelope>> {
959 self.events.subscribe()
960 }
961
962 async fn read<T: Send + 'static>(
963 &self,
964 query: impl FnOnce(&Connection) -> rusqlite::Result<T> + Send + 'static,
965 ) -> Result<T, StoreError> {
966 let reader = self.reader.clone();
967 Ok(tokio::task::spawn_blocking(move || query(&lock(&reader))).await??)
969 }
970
971 pub async fn jobs(&self, query: api::JobQuery) -> Result<api::JobList, StoreError> {
973 self.read(move |db| read::jobs(db, &query)).await
974 }
975
976 pub async fn timeline(&self, window: api::Span) -> Result<api::SlotTimeline, StoreError> {
978 let held: Vec<u64> = lock(&self.held).iter().map(|sample| sample.at).collect();
979 self.read(move |db| lens::timeline(db, window, &held, api::SAMPLE_INTERVAL))
980 .await
981 }
982
983 pub async fn compare(
985 &self,
986 id: u64,
987 with: Option<u64>,
988 ) -> Result<Option<api::JobComparison>, StoreError> {
989 self.read(move |db| lens::compare(db, id, with)).await
990 }
991
992 pub async fn job(&self, id: u64) -> Result<Option<api::Job>, StoreError> {
994 self.read(move |db| read::job(db, id)).await
995 }
996
997 pub async fn run_jobs(
999 &self,
1000 repository: &str,
1001 run: u64,
1002 except: u64,
1003 limit: usize,
1004 ) -> Result<RunJobs, StoreError> {
1005 let repository = repository.to_owned();
1006 self.read(move |db| run::run_jobs(db, &repository, run, except, limit))
1007 .await
1008 }
1009
1010 pub async fn job_samples(
1012 &self,
1013 id: u64,
1014 interval: Duration,
1015 ) -> Result<Option<api::SampleSeries>, StoreError> {
1016 self.read(move |db| read::job_samples(db, id, interval))
1017 .await
1018 }
1019
1020 pub async fn log_lines(
1022 &self,
1023 job: u64,
1024 from: u64,
1025 limit: u64,
1026 ) -> Result<Vec<(u64, String)>, StoreError> {
1027 self.read(move |db| read::log_lines(db, job, from, limit))
1028 .await
1029 }
1030
1031 pub async fn log_outline(&self, job: u64) -> Result<Option<api::LogOutline>, StoreError> {
1033 self.read(move |db| outline::outline(db, job)).await
1034 }
1035
1036 pub async fn styled_lines(
1039 &self,
1040 job: u64,
1041 from: u64,
1042 limit: u64,
1043 ) -> Result<Vec<api::StyledLine>, StoreError> {
1044 self.read(move |db| outline::styled_lines(db, job, from, limit))
1045 .await
1046 }
1047
1048 pub async fn search_log(
1051 &self,
1052 job: u64,
1053 mut search: api::Search,
1054 ) -> Result<api::LogSearch, StoreError> {
1055 while let Some(want) = search.wants() {
1057 search = self
1058 .read(move |db| {
1059 search.read(read::lines_from(db, job, want)?);
1060 Ok(search)
1061 })
1062 .await?;
1063 }
1064 Ok(search.finish())
1065 }
1066
1067 pub async fn events_after(
1069 &self,
1070 after: u64,
1071 limit: u64,
1072 ) -> Result<Vec<EventEnvelope>, StoreError> {
1073 self.read(move |db| read::events_after(db, after, limit))
1074 .await
1075 }
1076
1077 pub async fn sizing_changes(&self, since: u64) -> Result<Vec<api::SizingChange>, StoreError> {
1079 self.read(move |db| read::sizing_changes(db, since)).await
1080 }
1081
1082 pub async fn auto_sizing(&self) -> Result<AutoSizing, StoreError> {
1084 self.read(read::auto_sizing).await
1085 }
1086
1087 pub fn announced(&self) -> Result<Announced, StoreError> {
1090 Ok(read::announced(&lock(&self.reader))?)
1091 }
1092
1093 pub fn paused_repositories(&self) -> Result<Vec<String>, StoreError> {
1095 Ok(read::paused(&lock(&self.reader))?)
1096 }
1097
1098 pub async fn host_samples(&self, since: u64) -> Result<Vec<api::HostSample>, StoreError> {
1100 let held = lock(&self.held).clone();
1103 let mut samples = self.read(move |db| read::host_samples(db, since)).await?;
1104 let stored = samples.last().map_or(0, |sample| sample.at);
1105 samples.extend(
1106 held.into_iter()
1107 .filter(|sample| sample.at >= since && sample.at > stored),
1108 );
1109 Ok(samples)
1110 }
1111
1112 pub async fn activity(&self, since: u64, now: u64) -> Result<api::Activity, StoreError> {
1114 self.read(move |db| read::activity(db, since, now)).await
1115 }
1116
1117 pub async fn stats(&self, query: api::StatsQuery, now: u64) -> Result<api::Stats, StoreError> {
1119 self.read(move |db| read::stats(db, &query, now)).await
1120 }
1121
1122 pub async fn sizing(&self, since: u64) -> Result<sizing::History, StoreError> {
1124 self.read(move |db| read::sizing(db, since)).await
1125 }
1126
1127 pub async fn pricing(&self, since: u64) -> Result<api::PricingHistory, StoreError> {
1130 self.read(move |db| read::pricing(db, since)).await
1131 }
1132
1133 pub async fn insights(&self, since: u64) -> Result<crate::insights::History, StoreError> {
1136 let memory = self.latest_host_memory();
1137 let (waiting, seen) = lock(&self.host_waiting).clone();
1138 self.read(move |db| {
1139 let mut history = read::insights(db, since)?;
1140 if let Some(memory) = memory.filter(|_| !waiting.is_empty()) {
1141 let jobs = read::held_jobs(db, &waiting)?;
1142 history.host =
1143 (!jobs.is_empty()).then_some(crate::insights::HostHold { memory, seen, jobs });
1144 }
1145 Ok(history)
1146 })
1147 .await
1148 }
1149
1150 pub async fn last_event(&self) -> Result<u64, StoreError> {
1152 self.read(read::last_event).await
1153 }
1154
1155 pub async fn notify_state(
1157 &self,
1158 ) -> Result<BTreeMap<String, BTreeMap<String, u64>>, StoreError> {
1159 self.read(read::notify_state).await
1160 }
1161
1162 pub async fn warm(&self, since: u64) -> Result<Vec<warm::Seen>, StoreError> {
1164 self.read(move |db| read::warm(db, since)).await
1165 }
1166
1167 pub async fn cache_sizes_now(&self) -> Result<Measured, StoreError> {
1169 self.read(read::latest_cache_sizes).await
1170 }
1171
1172 pub async fn build_cache_since(&self, since: u64) -> Result<api::BuildCacheCounts, StoreError> {
1174 self.read(move |db| read::build_cache_since(db, since))
1175 .await
1176 }
1177
1178 pub async fn cache_history(&self, since: u64) -> Result<Vec<api::CacheSize>, StoreError> {
1180 self.read(move |db| read::cache_history(db, since)).await
1181 }
1182
1183 pub async fn open_jobs(&self) -> Result<Vec<OpenJob>, StoreError> {
1185 self.read(read::open_jobs).await
1186 }
1187
1188 pub async fn slot_jobs(&self) -> Result<Vec<SlotJob>, StoreError> {
1190 self.read(read::slot_jobs).await
1191 }
1192
1193 pub async fn last_runner_second(&self, prefix: String) -> Result<Option<u64>, StoreError> {
1197 self.read(move |db| read::last_runner_second(db, &prefix))
1198 .await
1199 }
1200
1201 pub async fn runner_job(&self, runner: &str) -> Result<Option<u64>, StoreError> {
1203 let runner = runner.to_owned();
1204 self.read(move |db| read::runner_job(db, &runner)).await
1205 }
1206
1207 pub async fn runner_admitted(
1209 &self,
1210 runner: &str,
1211 ) -> Result<Option<api::Admission>, StoreError> {
1212 let runner = runner.to_owned();
1213 self.read(move |db| read::runner_admission(db, &runner))
1214 .await
1215 }
1216
1217 pub async fn runner_cancel_deadline(&self, runner: &str) -> Result<Option<u64>, StoreError> {
1220 let runner = runner.to_owned();
1221 self.read(move |db| read::runner_cancel_deadline(db, &runner))
1222 .await
1223 }
1224
1225 pub async fn runner_memory_cap(&self, runner: &str) -> Result<Option<u64>, StoreError> {
1227 let runner = runner.to_owned();
1228 self.read(move |db| read::runner_memory_cap(db, &runner))
1229 .await
1230 }
1231
1232 pub async fn log_position(&self, job: u64) -> Result<LogPosition, StoreError> {
1234 self.read(move |db| read::log_position(db, job)).await
1235 }
1236
1237 pub async fn mirror_log_until(&self) -> Result<MirrorPosition, StoreError> {
1239 self.read(read::mirror_log_until).await
1240 }
1241}
1242
1243pub fn sizing_at(path: &Path, since: u64) -> Result<Option<sizing::History>, StoreError> {
1248 read_only(path, |db| sizing_of(db, since))
1249}
1250
1251pub fn trace_at(
1256 path: &Path,
1257 since: u64,
1258 named_by: u64,
1259 overhead: Option<u64>,
1260) -> Result<Option<Trace>, StoreError> {
1261 read_only(path, |db| {
1262 if version(db)? == 0 {
1263 return Ok(Trace::default());
1264 }
1265 Ok(read::trace(db, since, named_by, overhead)?)
1266 })
1267}
1268
1269pub fn last_queued_at(path: &Path) -> Result<Option<u64>, StoreError> {
1272 let last = read_only(path, |db| {
1273 if version(db)? == 0 {
1274 return Ok(None);
1275 }
1276 Ok(db.query_row("SELECT MAX(queued_at) FROM jobs", [], |row| row.get(0))?)
1277 })?;
1278 Ok(last.flatten())
1279}
1280
1281fn read_only<T>(
1283 path: &Path,
1284 read: impl Fn(&Connection) -> Result<T, StoreError>,
1285) -> Result<Option<T>, StoreError> {
1286 if !path.exists() {
1287 return Ok(None);
1288 }
1289 let read = |path: &str, flags| -> Result<T, StoreError> {
1290 let db = Connection::open_with_flags(path, flags)?;
1291 db.busy_timeout(Duration::from_secs(5))?;
1292 read(&db)
1293 };
1294 let name = path.to_string_lossy();
1295 match read(&name, OpenFlags::SQLITE_OPEN_READ_ONLY) {
1296 Err(StoreError::Sqlite(error))
1297 if matches!(
1298 error.sqlite_error_code(),
1299 Some(rusqlite::ErrorCode::ReadOnly | rusqlite::ErrorCode::CannotOpen)
1300 ) =>
1301 {
1302 let mut wal = path.as_os_str().to_owned();
1304 wal.push("-wal");
1305 let wal = PathBuf::from(wal);
1306 if std::fs::metadata(&wal).is_ok_and(|wal| wal.len() > 0) {
1307 return Err(StoreError::Unread(wal));
1308 }
1309 let uri = format!(
1310 "file:{}?mode=ro&immutable=1",
1311 name.replace('%', "%25")
1312 .replace('?', "%3f")
1313 .replace('#', "%23")
1314 );
1315 read(
1316 &uri,
1317 OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_URI,
1318 )
1319 }
1320 history => history,
1321 }
1322 .map(Some)
1323}
1324
1325fn sizing_of(db: &Connection, since: u64) -> Result<sizing::History, StoreError> {
1327 match version(db)? {
1328 0 => return Ok(sizing::History::default()),
1329 VERSION => {}
1330 _ => read_as_current(db)?,
1331 }
1332 let mut history = read::sizing(db, since)?;
1333 let auto = read::auto_sizing(db)?;
1334 history.memory_set = auto.memory;
1335 history.slots_set = auto.slots;
1336 history.changes = read::sizing_changes(db, since.saturating_sub(sizing::memory::HISTORY))?;
1337 Ok(history)
1338}
1339
1340fn version(db: &Connection) -> Result<u32, StoreError> {
1342 let found: u32 = db.pragma_query_value(None, "user_version", |row| row.get(0))?;
1343 match found {
1344 0 | OLDEST..=VERSION => Ok(found),
1345 found if found > VERSION => Err(StoreError::Newer {
1346 found,
1347 supported: VERSION,
1348 }),
1349 found => Err(StoreError::Older {
1350 found,
1351 oldest: OLDEST,
1352 }),
1353 }
1354}
1355
1356fn read_as_current(db: &Connection) -> Result<(), StoreError> {
1360 let current = Connection::open_in_memory()?;
1361 current.execute_batch(SCHEMA)?;
1362 let tables: Vec<(String, String)> = current
1363 .prepare(
1364 "SELECT name, sql FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'",
1365 )?
1366 .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?
1367 .collect::<rusqlite::Result<_>>()?;
1368 let columns =
1369 |db: &Connection, table: &str| -> rusqlite::Result<Vec<(String, Option<String>)>> {
1370 db.prepare("SELECT name, dflt_value FROM pragma_table_info(?1, 'main')")?
1371 .query_map([table], |row| Ok((row.get(0)?, row.get(1)?)))?
1372 .collect()
1373 };
1374 for (table, sql) in tables {
1375 let found: Vec<String> = columns(db, &table)?
1376 .into_iter()
1377 .map(|(name, _)| name)
1378 .collect();
1379 if found.is_empty() {
1380 db.execute_batch(&sql.replacen("CREATE TABLE", "CREATE TEMP TABLE", 1))?;
1381 continue;
1382 }
1383 let added: Vec<String> = columns(¤t, &table)?
1384 .into_iter()
1385 .filter(|(name, _)| !found.contains(name))
1386 .map(|(name, default)| format!("{} AS {name}", default.as_deref().unwrap_or("NULL")))
1387 .collect();
1388 if !added.is_empty() {
1389 db.execute_batch(&format!(
1390 "CREATE TEMP VIEW {table} AS SELECT *, {} FROM main.{table}",
1391 added.join(", ")
1392 ))?;
1393 }
1394 }
1395 Ok(())
1396}
1397
1398fn configure(db: &Connection) -> rusqlite::Result<()> {
1400 db.pragma_update(None, "journal_mode", "wal")?;
1401 db.pragma_update(None, "synchronous", "normal")?;
1402 db.pragma_update(None, "foreign_keys", "on")?;
1403 db.busy_timeout(Duration::from_secs(5))
1404}
1405
1406fn prepare(db: &mut Connection) -> Result<(), StoreError> {
1409 let found = version(db)?;
1410 if found == 0 {
1411 let transaction = db.transaction()?;
1412 transaction.execute_batch(SCHEMA)?;
1413 transaction.pragma_update(None, "user_version", VERSION)?;
1414 transaction.commit()?;
1415 return Ok(());
1416 }
1417 for (version, migration) in MIGRATIONS.iter().filter(|(version, _)| *version > found) {
1418 let transaction = db.transaction()?;
1419 transaction.execute_batch(migration)?;
1420 transaction.pragma_update(None, "user_version", version)?;
1421 transaction.commit()?;
1422 }
1423 Ok(())
1424}
1425
1426#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1431struct Downtime {
1432 from: u64,
1433 until: u64,
1434}
1435
1436impl Downtime {
1437 fn before(db: &mut Connection, now: u64) -> Result<Option<Downtime>, StoreError> {
1441 let transaction = db.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?;
1444 let last: Option<u64> = transaction.query_row(
1445 "SELECT max(at) FROM (SELECT max(at) AS at FROM host_samples UNION ALL SELECT max(at) FROM events \
1446 UNION ALL SELECT value FROM meta WHERE key = 'opened_at')",
1447 [],
1448 |row| row.get(0),
1449 )?;
1450 let downtime = last
1451 .filter(|last| *last < now)
1452 .map(|from| Downtime { from, until: now });
1453 if let Some(downtime) = downtime {
1454 let queued: Vec<(u64, u64)> = transaction
1455 .prepare("SELECT id, queued_at FROM jobs WHERE state = 'queued'")?
1456 .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?
1457 .collect::<rusqlite::Result<_>>()?;
1458 for (job, queued_at) in queued {
1459 transaction.execute(
1460 "UPDATE jobs SET unseen = unseen + ?2 WHERE id = ?1",
1461 rusqlite::params![job, downtime.of(queued_at)],
1462 )?;
1463 }
1464 }
1465 transaction.execute(
1466 "INSERT OR REPLACE INTO meta (key, value) VALUES ('opened_at', ?1)",
1467 [now],
1468 )?;
1469 transaction.commit()?;
1470 Ok(downtime)
1471 }
1472
1473 fn of(self, queued_at: u64) -> u64 {
1475 self.until.saturating_sub(queued_at.max(self.from))
1476 }
1477
1478 fn after(self, queued_at: u64, started_at: u64) -> u64 {
1481 self.of(queued_at)
1482 .saturating_sub(started_at.saturating_sub(queued_at.max(self.from)))
1483 }
1484}
1485
1486fn logged<T>(what: &str, result: rusqlite::Result<T>) -> Option<T> {
1488 result
1489 .map_err(|error| warn!("job history: cannot {what}: {error}"))
1490 .ok()
1491}
1492
1493#[cfg(test)]
1494mod tests;