1use std::collections::{BTreeMap, BTreeSet};
8use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt};
9use std::path::{Path, PathBuf};
10use std::sync::atomic::{AtomicBool, Ordering};
11use std::sync::{Arc, Mutex, TryLockError};
12use std::time::Duration;
13
14use rusqlite::{Connection, OpenFlags};
15use tokio::sync::{OwnedSemaphorePermit, Semaphore, broadcast, mpsc, oneshot};
16use tracing::warn;
17
18use crate::admission::trace::Trace;
19use crate::api::{self, Event, EventEnvelope};
20use crate::config::History;
21use crate::github::{JobDetails, NamedRun, QueuedJob};
22use crate::observe::disk::{Window, Written};
23use crate::observe::metrics::{CpuHeld, Events};
24use crate::scheduler::warm;
25use crate::sizing;
26use crate::sync::lock;
27
28mod lens;
29mod outline;
30mod prune;
31mod read;
32mod run;
33mod write;
34
35pub use run::RunJobs;
36
37const SCHEMA: &str = include_str!("store/schema.sql");
39
40const OLDEST: u32 = 15;
42
43const MIGRATIONS: &[(u32, &str)] = &[
46 (
47 16,
48 "ALTER TABLE jobs ADD COLUMN unseen INTEGER NOT NULL DEFAULT 0;",
49 ),
50 (
51 17,
52 "ALTER TABLE jobs ADD COLUMN cpu_periods INTEGER;\n\
53 ALTER TABLE jobs ADD COLUMN cpu_throttled INTEGER;\n\
54 ALTER TABLE jobs ADD COLUMN cpu_stalled INTEGER;\n\
55 ALTER TABLE samples ADD COLUMN cpu_periods INTEGER;\n\
56 ALTER TABLE samples ADD COLUMN cpu_throttled INTEGER;\n\
57 ALTER TABLE samples ADD COLUMN cpu_stalled INTEGER;",
58 ),
59 (
60 18,
61 "ALTER TABLE jobs ADD COLUMN slot_waited INTEGER NOT NULL DEFAULT 0;",
62 ),
63 (
64 19,
65 "ALTER TABLE jobs ADD COLUMN seen_at INTEGER;\n\
66 ALTER TABLE runners ADD COLUMN purpose TEXT;\n\
67 ALTER TABLE runners ADD COLUMN stopped_for TEXT;",
68 ),
69 (
70 20,
71 "ALTER TABLE jobs ADD COLUMN runner_fault INTEGER NOT NULL DEFAULT 0;",
72 ),
73 (21, "ALTER TABLE runners ADD COLUMN cancel_by INTEGER;"),
74 (
75 22,
76 "CREATE TABLE log_outlines (job INTEGER PRIMARY KEY NOT NULL, from_line INTEGER NOT NULL, \
77 errors INTEGER NOT NULL, warnings INTEGER NOT NULL, groups_kept INTEGER NOT NULL, \
78 marks_kept INTEGER NOT NULL, sections_kept INTEGER NOT NULL, dropped INTEGER NOT NULL, \
79 hidden INTEGER NOT NULL);\n\
80 CREATE TABLE log_groups (job INTEGER NOT NULL, start INTEGER NOT NULL, end_line INTEGER, \
81 depth INTEGER NOT NULL, title TEXT NOT NULL, PRIMARY KEY (job, start)) WITHOUT ROWID;\n\
82 CREATE INDEX log_groups_open ON log_groups (job, start) WHERE end_line IS NULL;\n\
83 CREATE TABLE log_marks (job INTEGER NOT NULL, line INTEGER NOT NULL, kind TEXT NOT NULL, \
84 PRIMARY KEY (job, line)) WITHOUT ROWID;",
85 ),
86 (
87 23,
88 "CREATE TABLE sessions (id INTEGER PRIMARY KEY, started_at INTEGER NOT NULL, sampled_at INTEGER);\n\
89 ALTER TABLE records ADD COLUMN gone_at INTEGER;",
90 ),
91 (
92 24,
93 "ALTER TABLE jobs ADD COLUMN disk_written INTEGER;\n\
94 ALTER TABLE jobs ADD COLUMN disk_write_ios INTEGER;\n\
95 ALTER TABLE jobs ADD COLUMN disk_dirty INTEGER;\n\
96 ALTER TABLE jobs ADD COLUMN disk_measure TEXT;\n\
97 ALTER TABLE runners ADD COLUMN cgroup_gone_at INTEGER;\n\
98 ALTER TABLE samples ADD COLUMN written INTEGER;\n\
99 ALTER TABLE samples ADD COLUMN write_ios INTEGER;\n\
100 ALTER TABLE samples ADD COLUMN dirty INTEGER;\n\
101 CREATE TABLE disk_windows (start INTEGER PRIMARY KEY NOT NULL, end_at INTEGER NOT NULL, \
102 device TEXT NOT NULL, store TEXT, drive INTEGER, slice INTEGER, daemon INTEGER, commits INTEGER) WITHOUT ROWID;\n\
103 CREATE INDEX jobs_by_runner ON jobs (runner);",
104 ),
105];
106
107const VERSION: u32 = match MIGRATIONS.last() {
109 Some((version, _)) => *version,
110 None => OLDEST,
111};
112
113pub(crate) const LOG_BACKLOG: u64 = 8 << 20;
116
117const HOST_SAMPLE_BATCH: usize = 30;
120
121const EVENT_BUFFER: usize = 4096;
123
124#[derive(Debug, thiserror::Error)]
126pub enum StoreError {
127 #[error("{0}")]
129 Sqlite(#[from] rusqlite::Error),
130 #[error(
132 "the job history database is version {found}, newer than the {supported} this runnerd reads; \
133 run a newer runnerd or move the database aside"
134 )]
135 Newer {
136 found: u32,
138 supported: u32,
140 },
141 #[error(
143 "the job history database is version {found}, from a runnerd before 0.0.19; this runnerd reads versions \
144 from {oldest} on: move the database aside"
145 )]
146 Older {
147 found: u32,
149 oldest: u32,
151 },
152 #[error("{0}")]
154 Io(#[from] std::io::Error),
155 #[error("a database read stopped: {0}")]
157 Join(#[from] tokio::task::JoinError),
158 #[error(
161 "{} holds changes not yet in the job history, which this user cannot read; run runnerd doctor as the user \
162 the daemon runs as",
163 .0.display()
164 )]
165 Unread(PathBuf),
166}
167
168impl StoreError {
169 pub fn is_shutdown(&self) -> bool {
172 matches!(self, StoreError::Join(join) if join.is_cancelled())
173 }
174}
175
176#[derive(Debug, Clone, PartialEq, Eq)]
178pub struct RunnerRecord {
179 pub name: String,
181 pub repository: String,
183 pub profile: String,
185 pub slot: usize,
187 pub container: String,
189 pub started_at: u64,
191 pub memory_cap: Option<u64>,
193 pub cpu_cap: Option<u64>,
195 pub admission: Option<api::Admission>,
197 pub purpose: Option<&'static str>,
199}
200
201#[derive(Debug, Clone, PartialEq, Eq)]
203pub struct SampleRecord {
204 pub runner: String,
206 pub at: u64,
208 pub cpu: u64,
210 pub cpu_usage: u64,
212 pub memory: u64,
214 pub network_received: u64,
216 pub network_sent: u64,
218 pub events: Events,
220 pub cpu_held: CpuHeld,
222 pub written: Option<Written>,
224 pub dirty: Option<u64>,
226}
227
228#[derive(Debug, Clone, Default, PartialEq, Eq)]
230pub struct Measured {
231 pub at: Option<u64>,
233 pub bytes: BTreeMap<String, u64>,
235}
236
237#[derive(Debug, Clone, Default, PartialEq, Eq)]
240pub struct Announced {
241 pub profiles: Vec<api::ProfileStatus>,
243 pub repositories: Vec<api::RepositoryStatus>,
245}
246
247#[derive(Debug, Clone, PartialEq, Eq)]
249pub struct OpenJob {
250 pub id: u64,
252 pub repository: String,
254 pub state: api::JobState,
256 pub runner: Option<String>,
258 pub runner_removed_at: Option<u64>,
260 pub run_id: u64,
262 pub run_attempt: u32,
264 pub workflow: Option<String>,
266 pub queued_at: u64,
268 pub labels: Vec<String>,
270}
271
272#[derive(Debug, Clone, PartialEq, Eq)]
274pub struct SlotJob {
275 pub profile: String,
277 pub slot: usize,
279 pub key: crate::admission::Key,
281}
282
283#[derive(Debug, Clone, Copy, PartialEq, Eq)]
285pub struct LogPosition {
286 pub page: u64,
288 pub offset: u64,
290}
291
292#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
296pub struct MirrorPosition {
297 pub at: u64,
299 pub seen: u64,
301}
302
303#[derive(Debug)]
305enum Write {
306 Queued {
307 repository: String,
308 job: QueuedJob,
309 seen_at: u64,
310 },
311 RunnerStarted(RunnerRecord),
312 RunnerAdmission {
313 runner: String,
314 admission: api::Admission,
315 },
316 RunnerOomKilled(String),
317 RunnerStoppedFor {
318 runner: String,
319 reason: &'static str,
320 },
321 RunnerCancelBy {
322 runner: String,
323 at: u64,
324 },
325 HostWaited {
326 jobs: Vec<u64>,
327 ms: u64,
328 },
329 SlotWaited {
330 jobs: Vec<u64>,
331 ms: u64,
332 },
333 BuildCache {
334 runner: String,
335 counts: api::BuildCacheCounts,
336 },
337 RunnerRemoved {
338 runner: String,
339 at: u64,
340 },
341 RunnersGone {
342 running: Vec<String>,
343 at: u64,
344 },
345 Claimed {
346 job: u64,
347 run_id: u64,
348 runner: String,
349 at: u64,
350 },
351 Details {
352 details: JobDetails,
353 },
354 WorkflowNamed(NamedRun),
355 Elsewhere {
356 details: JobDetails,
357 },
358 Lost {
359 job: u64,
360 at: u64,
361 },
362 Unanswered {
363 job: u64,
364 at: u64,
365 },
366 Log {
367 job: u64,
368 lines: Vec<String>,
369 position: Option<LogPosition>,
371 max_bytes: u64,
372 _backlog: OwnedSemaphorePermit,
374 },
375 Samples(Vec<SampleRecord>),
376 CgroupsGone {
377 runners: Vec<String>,
378 at: u64,
379 },
380 DiskWindow(Window),
381 HostSample(api::HostSample),
382 MirrorBytes {
383 bytes: Vec<(String, u64)>,
384 until: MirrorPosition,
385 done: oneshot::Sender<bool>,
387 },
388 CacheSizes {
389 at: u64,
390 sizes: BTreeMap<String, u64>,
391 },
392 Event(Box<Event>),
394 RecordMade(api::Record),
395 RecordSkipped {
396 job: u64,
397 why: String,
398 },
399 RecordRemoved {
400 job: u64,
401 at: u64,
402 by: api::RecordRemoval,
403 },
404 RecordsListed {
405 jobs: Vec<u64>,
406 at: u64,
407 },
408 Paused {
409 repositories: Vec<String>,
410 done: oneshot::Sender<bool>,
412 },
413 Visibility {
414 repository: String,
415 visibility: api::Visibility,
416 at: u64,
417 },
418 SizingIntent {
419 change: api::SizingChange,
420 done: oneshot::Sender<Option<i64>>,
422 },
423 SizingResolved {
424 id: i64,
425 resolution: Resolution,
426 done: oneshot::Sender<bool>,
428 },
429 Notify {
430 change: NotifyChange,
431 done: oneshot::Sender<bool>,
433 },
434 RunnerNamed {
435 secs: u64,
436 done: oneshot::Sender<bool>,
438 },
439 Flush(oneshot::Sender<()>),
440}
441
442#[derive(Debug)]
444enum NotifyChange {
445 Record {
447 target: String,
448 set: Vec<(String, u64)>,
449 remove: Vec<String>,
450 },
451 Keep(Vec<String>),
453}
454
455#[derive(Debug, Clone, Copy, PartialEq, Eq)]
457pub enum Resolution {
458 Done,
460 Failed,
462 Uncertain,
465}
466
467#[derive(Debug, Clone, Default, PartialEq, Eq)]
469pub struct AutoSizing {
470 pub last: BTreeMap<String, u64>,
472 pub pending: Vec<(i64, api::SizingChange)>,
474 pub cpus: Vec<sizing::CpusSet>,
477 pub memory: Vec<sizing::MemorySet>,
479 pub slots: Vec<sizing::SlotsSet>,
482}
483
484#[derive(Debug, Clone)]
486pub struct Store {
487 writes: mpsc::UnboundedSender<Write>,
488 reader: Arc<Mutex<Connection>>,
489 pruner: Arc<Mutex<Connection>>,
491 closed: Arc<AtomicBool>,
493 held: Arc<Mutex<Vec<api::HostSample>>>,
496 latest_host: Arc<Mutex<Option<api::HostSample>>>,
498 latest_host_memory: Arc<Mutex<Option<api::HostMemory>>>,
500 host_waiting: Arc<Mutex<(BTreeSet<u64>, u64)>>,
502 latest_samples: Arc<Mutex<BTreeMap<String, SampleRecord>>>,
504 gone: Arc<Mutex<BTreeSet<String>>>,
506 events: broadcast::Sender<Arc<EventEnvelope>>,
507 backlog: Arc<Semaphore>,
508}
509
510impl Store {
511 pub fn open(path: &Path) -> Result<Store, StoreError> {
516 Store::open_at(path, crate::time::now_ms())
517 }
518
519 fn open_at(path: &Path, now: u64) -> Result<Store, StoreError> {
521 if let Some(dir) = path.parent().filter(|dir| !dir.as_os_str().is_empty()) {
522 std::fs::DirBuilder::new()
523 .recursive(true)
524 .mode(0o700)
525 .create(dir)?;
526 }
527 std::fs::OpenOptions::new()
529 .create(true)
530 .append(true)
531 .mode(0o600)
532 .open(path)?;
533 let mut db = Connection::open(path)?;
534 configure(&db)?;
535 prepare(&mut db)?;
536 let downtime = Downtime::before(&mut db, now)?;
537 db.execute("INSERT INTO sessions (started_at) VALUES (?1)", [now])?;
538 let session = db.last_insert_rowid();
539 let reader = Connection::open_with_flags(
540 path,
541 OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX,
542 )?;
543 configure(&reader)?;
544 let pruner = Connection::open(path)?;
545 configure(&pruner)?;
546 let (events, _) = broadcast::channel(EVENT_BUFFER);
547 let (writes, receiver) = mpsc::unbounded_channel();
548 let thread_events = events.clone();
549 let held = Arc::new(Mutex::new(Vec::new()));
550 let thread_held = held.clone();
551 std::thread::Builder::new()
552 .name("runnerd-store".to_owned())
553 .spawn(move || {
554 write::run(
555 write::Started {
556 db,
557 downtime,
558 session,
559 },
560 receiver,
561 &thread_events,
562 &thread_held,
563 );
564 })?;
565 Ok(Store {
566 writes,
567 reader: Arc::new(Mutex::new(reader)),
568 pruner: Arc::new(Mutex::new(pruner)),
569 closed: Arc::new(AtomicBool::new(false)),
570 held,
571 latest_host: Arc::new(Mutex::new(None)),
572 latest_host_memory: Arc::new(Mutex::new(None)),
573 host_waiting: Arc::new(Mutex::new((BTreeSet::new(), 0))),
574 latest_samples: Arc::new(Mutex::new(BTreeMap::new())),
575 gone: Arc::new(Mutex::new(BTreeSet::new())),
576 events,
577 backlog: Arc::new(Semaphore::new(
578 usize::try_from(LOG_BACKLOG).unwrap_or(usize::MAX),
579 )),
580 })
581 }
582
583 fn send(&self, write: Write) {
584 let _ = self.writes.send(write);
586 }
587
588 pub fn queued(&self, repository: &str, job: &QueuedJob, seen_at: u64) {
592 self.send(Write::Queued {
593 repository: repository.to_owned(),
594 job: job.clone(),
595 seen_at,
596 });
597 }
598
599 pub fn runner_started(&self, runner: RunnerRecord) {
601 self.send(Write::RunnerStarted(runner));
602 }
603
604 pub fn runner_removed(&self, runner: &str, at: u64) {
606 lock(&self.latest_samples).remove(runner);
607 lock(&self.gone).remove(runner);
608 self.send(Write::RunnerRemoved {
609 runner: runner.to_owned(),
610 at,
611 });
612 }
613
614 pub fn runners_gone(&self, running: Vec<String>, at: u64) {
617 lock(&self.latest_samples).retain(|runner, _| running.contains(runner));
618 lock(&self.gone).retain(|runner| running.contains(runner));
619 self.send(Write::RunnersGone { running, at });
620 }
621
622 pub fn claimed(&self, job: u64, run_id: u64, runner: &str, at: u64) {
624 self.send(Write::Claimed {
625 job,
626 run_id,
627 runner: runner.to_owned(),
628 at,
629 });
630 }
631
632 pub fn details(&self, details: JobDetails) {
634 self.send(Write::Details { details });
635 }
636
637 pub fn workflow_named(&self, named: NamedRun) {
640 self.send(Write::WorkflowNamed(named));
641 }
642
643 pub fn elsewhere(&self, details: JobDetails) {
646 self.send(Write::Elsewhere { details });
647 }
648
649 pub fn unanswered(&self, job: u64, at: u64) {
652 self.send(Write::Unanswered { job, at });
653 }
654
655 pub fn lost(&self, job: u64, at: u64) {
657 self.send(Write::Lost { job, at });
658 }
659
660 pub async fn log(&self, job: u64, lines: Vec<String>, position: LogPosition, max_bytes: u64) {
663 let size = |line: &String| line.len() as u64 + 1;
664 let mut lines = lines.into_iter().map(|mut line| {
665 if size(&line) > LOG_BACKLOG {
667 let mut end = usize::try_from(LOG_BACKLOG / 2).unwrap_or(usize::MAX);
668 while !line.is_char_boundary(end) {
669 end -= 1;
670 }
671 line.truncate(end);
672 }
673 line
674 });
675 let mut next = lines.next();
676 loop {
677 let (mut part, mut bytes) = (Vec::new(), 0);
678 while let Some(line) = next.take_if(|line| bytes + size(line) <= LOG_BACKLOG) {
679 bytes += size(&line);
680 part.push(line);
681 next = lines.next();
682 }
683 let last = next.is_none();
684 let share = u32::try_from(bytes.max(1)).unwrap_or(u32::MAX);
685 let Ok(backlog) = self.backlog.clone().acquire_many_owned(share).await else {
687 return;
688 };
689 self.send(Write::Log {
690 job,
691 lines: part,
692 position: last.then_some(position),
693 max_bytes,
694 _backlog: backlog,
695 });
696 if last {
697 return;
698 }
699 }
700 }
701
702 pub fn samples(&self, samples: Vec<SampleRecord>) {
704 let mut latest = lock(&self.latest_samples);
705 for sample in &samples {
706 latest.insert(sample.runner.clone(), sample.clone());
707 }
708 drop(latest);
709 self.send(Write::Samples(samples));
710 }
711
712 pub fn host_sample(&self, sample: api::HostSample) {
715 *lock(&self.latest_host) = Some(sample.clone());
716 lock(&self.held).push(sample.clone());
717 self.send(Write::HostSample(sample));
718 }
719
720 pub fn latest_sample(&self, runner: &str) -> Option<SampleRecord> {
722 lock(&self.latest_samples).get(runner).cloned()
723 }
724
725 pub fn containers_gone(&self, runners: Vec<String>) {
728 lock(&self.gone).extend(runners);
729 }
730
731 pub fn containers_ended(&self, runners: Vec<String>, at: u64) {
735 if !runners.is_empty() {
736 self.send(Write::CgroupsGone { runners, at });
737 }
738 }
739
740 pub fn disk_window(&self, window: Window) {
742 self.send(Write::DiskWindow(window));
743 }
744
745 pub fn containers_read(&self, runners: &[String]) {
748 let mut gone = lock(&self.gone);
749 for runner in runners {
750 gone.remove(runner);
751 }
752 }
753
754 pub fn container_gone(&self, runner: &str) -> bool {
756 lock(&self.gone).contains(runner)
757 }
758
759 pub fn runner_admission(&self, runner: &str, admission: api::Admission) {
761 self.send(Write::RunnerAdmission {
762 runner: runner.to_owned(),
763 admission,
764 });
765 }
766
767 pub fn runner_oom_killed(&self, runner: &str) {
770 self.send(Write::RunnerOomKilled(runner.to_owned()));
771 }
772
773 pub fn runner_cancel_by(&self, runner: &str, at: u64) {
776 self.send(Write::RunnerCancelBy {
777 runner: runner.to_owned(),
778 at,
779 });
780 }
781
782 pub fn runner_stopped_for(&self, runner: &str, reason: &'static str) {
784 self.send(Write::RunnerStoppedFor {
785 runner: runner.to_owned(),
786 reason,
787 });
788 }
789
790 pub fn build_cache(&self, runner: &str, counts: api::BuildCacheCounts) {
792 self.send(Write::BuildCache {
793 runner: runner.to_owned(),
794 counts,
795 });
796 }
797
798 pub fn host_memory(&self, memory: api::HostMemory) {
800 *lock(&self.latest_host_memory) = Some(memory);
801 }
802
803 pub fn latest_host_memory(&self) -> Option<api::HostMemory> {
805 lock(&self.latest_host_memory).clone()
806 }
807
808 pub fn host_waiting(&self, jobs: BTreeSet<u64>, at: u64) {
811 *lock(&self.host_waiting) = (jobs, at);
812 }
813
814 pub fn host_waited(&self, jobs: Vec<u64>, ms: u64) {
817 self.send(Write::HostWaited { jobs, ms });
818 }
819
820 pub fn slot_waited(&self, jobs: Vec<u64>, ms: u64) {
823 self.send(Write::SlotWaited { jobs, ms });
824 }
825
826 pub fn latest_host_sample(&self) -> Option<api::HostSample> {
828 lock(&self.latest_host).clone()
829 }
830
831 pub async fn mirror_bytes(&self, bytes: Vec<(String, u64)>, until: MirrorPosition) -> bool {
834 let (done, committed) = oneshot::channel();
835 self.send(Write::MirrorBytes { bytes, until, done });
836 committed.await.unwrap_or(false)
837 }
838
839 pub fn cache_sizes(&self, at: u64, sizes: BTreeMap<String, u64>) {
841 self.send(Write::CacheSizes { at, sizes });
842 }
843
844 pub fn event(&self, event: Event) {
846 self.send(Write::Event(Box::new(event)));
847 }
848
849 pub fn record_made(&self, record: api::Record) {
851 self.send(Write::RecordMade(record));
852 }
853
854 pub fn record_skipped(&self, job: u64, why: String) {
856 self.send(Write::RecordSkipped { job, why });
857 }
858
859 pub fn record_removed(&self, job: u64, at: u64, by: api::RecordRemoval) {
861 self.send(Write::RecordRemoved { job, at, by });
862 }
863
864 pub fn records_listed(&self, jobs: Vec<u64>, at: u64) {
868 self.send(Write::RecordsListed { jobs, at });
869 }
870
871 pub fn paused(&self, repositories: Vec<String>) -> impl Future<Output = bool> + use<> {
874 let (done, committed) = oneshot::channel();
875 self.send(Write::Paused { repositories, done });
876 async move { committed.await.unwrap_or(false) }
877 }
878
879 pub async fn runner_named(&self, secs: u64) -> bool {
883 let (done, committed) = oneshot::channel();
884 self.send(Write::RunnerNamed { secs, done });
885 tokio::task::spawn_blocking(move || committed.blocking_recv().unwrap_or(false))
888 .await
889 .unwrap_or(false)
890 }
891
892 pub fn visibility(&self, repository: String, visibility: api::Visibility, at: u64) {
894 self.send(Write::Visibility {
895 repository,
896 visibility,
897 at,
898 });
899 }
900
901 pub fn sizing_intent(
904 &self,
905 change: api::SizingChange,
906 ) -> impl Future<Output = Option<i64>> + use<> {
907 let (done, committed) = oneshot::channel();
908 self.send(Write::SizingIntent { change, done });
909 async move { committed.await.ok().flatten() }
910 }
911
912 pub fn sizing_resolved(
915 &self,
916 id: i64,
917 resolution: Resolution,
918 ) -> impl Future<Output = bool> + use<> {
919 let (done, committed) = oneshot::channel();
920 self.send(Write::SizingResolved {
921 id,
922 resolution,
923 done,
924 });
925 async move { committed.await.unwrap_or(false) }
926 }
927
928 pub fn notify_record(
931 &self,
932 target: &str,
933 set: Vec<(String, u64)>,
934 remove: Vec<String>,
935 ) -> impl Future<Output = bool> + use<> {
936 let (done, committed) = oneshot::channel();
937 self.send(Write::Notify {
938 change: NotifyChange::Record {
939 target: target.to_owned(),
940 set,
941 remove,
942 },
943 done,
944 });
945 async move { committed.await.unwrap_or(false) }
946 }
947
948 pub fn notify_keep(&self, targets: Vec<String>) -> impl Future<Output = bool> + use<> {
950 let (done, committed) = oneshot::channel();
951 self.send(Write::Notify {
952 change: NotifyChange::Keep(targets),
953 done,
954 });
955 async move { committed.await.unwrap_or(false) }
956 }
957
958 pub fn prune(&self, now: u64, history: &History) -> tokio::task::JoinHandle<()> {
962 let pruner = self.pruner.clone();
963 let closed = self.closed.clone();
964 let history = history.clone();
965 tokio::task::spawn_blocking(move || {
966 let db = match pruner.try_lock() {
967 Ok(db) => db,
968 Err(TryLockError::Poisoned(poisoned)) => poisoned.into_inner(),
969 Err(TryLockError::WouldBlock) => return,
970 };
971 let db = prune::Db {
972 connection: &db,
973 stop: &closed,
974 };
975 if !db.stop.load(Ordering::Relaxed) {
976 prune::run(&db, now, &history);
977 }
978 })
979 }
980
981 pub fn close(&self) {
984 self.closed.store(true, Ordering::Relaxed);
985 }
986
987 pub async fn flush(&self) {
989 let (done, wait) = oneshot::channel();
990 self.send(Write::Flush(done));
991 let _ = wait.await;
992 }
993
994 pub fn subscribe(&self) -> broadcast::Receiver<Arc<EventEnvelope>> {
997 self.events.subscribe()
998 }
999
1000 async fn read<T: Send + 'static>(
1001 &self,
1002 query: impl FnOnce(&Connection) -> rusqlite::Result<T> + Send + 'static,
1003 ) -> Result<T, StoreError> {
1004 let reader = self.reader.clone();
1005 Ok(tokio::task::spawn_blocking(move || query(&lock(&reader))).await??)
1007 }
1008
1009 pub async fn jobs(&self, query: api::JobQuery) -> Result<api::JobList, StoreError> {
1011 self.read(move |db| read::jobs(db, &query)).await
1012 }
1013
1014 pub async fn timeline(&self, window: api::Span) -> Result<api::SlotTimeline, StoreError> {
1016 let held: Vec<u64> = lock(&self.held).iter().map(|sample| sample.at).collect();
1017 self.read(move |db| lens::timeline(db, window, &held, api::SAMPLE_INTERVAL))
1018 .await
1019 }
1020
1021 pub async fn compare(
1023 &self,
1024 id: u64,
1025 with: Option<u64>,
1026 ) -> Result<Option<api::JobComparison>, StoreError> {
1027 self.read(move |db| lens::compare(db, id, with)).await
1028 }
1029
1030 pub async fn job(&self, id: u64) -> Result<Option<api::Job>, StoreError> {
1032 self.read(move |db| read::job(db, id)).await
1033 }
1034
1035 pub async fn run_jobs(
1037 &self,
1038 repository: &str,
1039 run: u64,
1040 except: u64,
1041 limit: usize,
1042 ) -> Result<RunJobs, StoreError> {
1043 let repository = repository.to_owned();
1044 self.read(move |db| run::run_jobs(db, &repository, run, except, limit))
1045 .await
1046 }
1047
1048 pub async fn job_samples(
1050 &self,
1051 id: u64,
1052 interval: Duration,
1053 ) -> Result<Option<api::SampleSeries>, StoreError> {
1054 self.read(move |db| read::job_samples(db, id, interval))
1055 .await
1056 }
1057
1058 pub async fn log_lines(
1060 &self,
1061 job: u64,
1062 from: u64,
1063 limit: u64,
1064 ) -> Result<Vec<(u64, String)>, StoreError> {
1065 self.read(move |db| read::log_lines(db, job, from, limit))
1066 .await
1067 }
1068
1069 pub async fn log_outline(&self, job: u64) -> Result<Option<api::LogOutline>, StoreError> {
1071 self.read(move |db| outline::outline(db, job)).await
1072 }
1073
1074 pub async fn styled_lines(
1077 &self,
1078 job: u64,
1079 from: u64,
1080 limit: u64,
1081 ) -> Result<Vec<api::StyledLine>, StoreError> {
1082 self.read(move |db| outline::styled_lines(db, job, from, limit))
1083 .await
1084 }
1085
1086 pub async fn search_log(
1089 &self,
1090 job: u64,
1091 mut search: api::Search,
1092 ) -> Result<api::LogSearch, StoreError> {
1093 while let Some(want) = search.wants() {
1095 search = self
1096 .read(move |db| {
1097 search.read(read::lines_from(db, job, want)?);
1098 Ok(search)
1099 })
1100 .await?;
1101 }
1102 Ok(search.finish())
1103 }
1104
1105 pub async fn events_after(
1107 &self,
1108 after: u64,
1109 limit: u64,
1110 ) -> Result<Vec<EventEnvelope>, StoreError> {
1111 self.read(move |db| read::events_after(db, after, limit))
1112 .await
1113 }
1114
1115 pub async fn sizing_changes(&self, since: u64) -> Result<Vec<api::SizingChange>, StoreError> {
1117 self.read(move |db| read::sizing_changes(db, since)).await
1118 }
1119
1120 pub async fn auto_sizing(&self) -> Result<AutoSizing, StoreError> {
1122 self.read(read::auto_sizing).await
1123 }
1124
1125 pub fn announced(&self) -> Result<Announced, StoreError> {
1128 Ok(read::announced(&lock(&self.reader))?)
1129 }
1130
1131 pub fn paused_repositories(&self) -> Result<Vec<String>, StoreError> {
1133 Ok(read::paused(&lock(&self.reader))?)
1134 }
1135
1136 pub async fn host_samples(&self, since: u64) -> Result<Vec<api::HostSample>, StoreError> {
1138 let held = lock(&self.held).clone();
1141 let mut samples = self.read(move |db| read::host_samples(db, since)).await?;
1142 let stored = samples.last().map_or(0, |sample| sample.at);
1143 samples.extend(
1144 held.into_iter()
1145 .filter(|sample| sample.at >= since && sample.at > stored),
1146 );
1147 Ok(samples)
1148 }
1149
1150 pub async fn activity(&self, since: u64, now: u64) -> Result<api::Activity, StoreError> {
1152 self.read(move |db| read::activity(db, since, now)).await
1153 }
1154
1155 pub async fn stats(&self, query: api::StatsQuery, now: u64) -> Result<api::Stats, StoreError> {
1157 self.read(move |db| read::stats(db, &query, now)).await
1158 }
1159
1160 pub async fn sizing(&self, since: u64) -> Result<sizing::History, StoreError> {
1162 self.read(move |db| read::sizing(db, since)).await
1163 }
1164
1165 pub async fn pricing(&self, since: u64) -> Result<api::PricingHistory, StoreError> {
1168 self.read(move |db| read::pricing(db, since)).await
1169 }
1170
1171 pub async fn insights(&self, since: u64) -> Result<crate::insights::History, StoreError> {
1174 let memory = self.latest_host_memory();
1175 let (waiting, seen) = lock(&self.host_waiting).clone();
1176 self.read(move |db| {
1177 let mut history = read::insights(db, since)?;
1178 if let Some(memory) = memory.filter(|_| !waiting.is_empty()) {
1179 let jobs = read::held_jobs(db, &waiting)?;
1180 history.host =
1181 (!jobs.is_empty()).then_some(crate::insights::HostHold { memory, seen, jobs });
1182 }
1183 Ok(history)
1184 })
1185 .await
1186 }
1187
1188 pub async fn last_event(&self) -> Result<u64, StoreError> {
1190 self.read(read::last_event).await
1191 }
1192
1193 pub async fn notify_state(
1195 &self,
1196 ) -> Result<BTreeMap<String, BTreeMap<String, u64>>, StoreError> {
1197 self.read(read::notify_state).await
1198 }
1199
1200 pub async fn warm(&self, since: u64) -> Result<Vec<warm::Seen>, StoreError> {
1202 self.read(move |db| read::warm(db, since)).await
1203 }
1204
1205 pub async fn cache_sizes_now(&self) -> Result<Measured, StoreError> {
1207 self.read(read::latest_cache_sizes).await
1208 }
1209
1210 pub async fn build_cache_since(&self, since: u64) -> Result<api::BuildCacheCounts, StoreError> {
1212 self.read(move |db| read::build_cache_since(db, since))
1213 .await
1214 }
1215
1216 pub async fn cache_history(&self, since: u64) -> Result<Vec<api::CacheSize>, StoreError> {
1218 self.read(move |db| read::cache_history(db, since)).await
1219 }
1220
1221 pub async fn open_jobs(&self) -> Result<Vec<OpenJob>, StoreError> {
1223 self.read(read::open_jobs).await
1224 }
1225
1226 pub async fn slot_jobs(&self) -> Result<Vec<SlotJob>, StoreError> {
1228 self.read(read::slot_jobs).await
1229 }
1230
1231 pub async fn last_runner_second(&self, prefix: String) -> Result<Option<u64>, StoreError> {
1235 self.read(move |db| read::last_runner_second(db, &prefix))
1236 .await
1237 }
1238
1239 pub async fn runner_job(&self, runner: &str) -> Result<Option<u64>, StoreError> {
1241 let runner = runner.to_owned();
1242 self.read(move |db| read::runner_job(db, &runner)).await
1243 }
1244
1245 pub async fn runner_admitted(
1247 &self,
1248 runner: &str,
1249 ) -> Result<Option<api::Admission>, StoreError> {
1250 let runner = runner.to_owned();
1251 self.read(move |db| read::runner_admission(db, &runner))
1252 .await
1253 }
1254
1255 pub async fn runner_cancel_deadline(&self, runner: &str) -> Result<Option<u64>, StoreError> {
1258 let runner = runner.to_owned();
1259 self.read(move |db| read::runner_cancel_deadline(db, &runner))
1260 .await
1261 }
1262
1263 pub async fn runner_memory_cap(&self, runner: &str) -> Result<Option<u64>, StoreError> {
1265 let runner = runner.to_owned();
1266 self.read(move |db| read::runner_memory_cap(db, &runner))
1267 .await
1268 }
1269
1270 pub async fn log_position(&self, job: u64) -> Result<LogPosition, StoreError> {
1272 self.read(move |db| read::log_position(db, job)).await
1273 }
1274
1275 pub async fn mirror_log_until(&self) -> Result<MirrorPosition, StoreError> {
1277 self.read(read::mirror_log_until).await
1278 }
1279}
1280
1281pub fn sizing_at(path: &Path, since: u64) -> Result<Option<sizing::History>, StoreError> {
1286 read_only(path, |db| sizing_of(db, since))
1287}
1288
1289pub fn trace_at(
1294 path: &Path,
1295 since: u64,
1296 named_by: u64,
1297 overhead: Option<u64>,
1298) -> Result<Option<Trace>, StoreError> {
1299 read_only(path, |db| {
1300 if version(db)? == 0 {
1301 return Ok(Trace::default());
1302 }
1303 Ok(read::trace(db, since, named_by, overhead)?)
1304 })
1305}
1306
1307pub fn last_queued_at(path: &Path) -> Result<Option<u64>, StoreError> {
1310 let last = read_only(path, |db| {
1311 if version(db)? == 0 {
1312 return Ok(None);
1313 }
1314 Ok(db.query_row("SELECT MAX(queued_at) FROM jobs", [], |row| row.get(0))?)
1315 })?;
1316 Ok(last.flatten())
1317}
1318
1319fn read_only<T>(
1321 path: &Path,
1322 read: impl Fn(&Connection) -> Result<T, StoreError>,
1323) -> Result<Option<T>, StoreError> {
1324 if !path.exists() {
1325 return Ok(None);
1326 }
1327 let read = |path: &str, flags| -> Result<T, StoreError> {
1328 let db = Connection::open_with_flags(path, flags)?;
1329 db.busy_timeout(Duration::from_secs(5))?;
1330 read(&db)
1331 };
1332 let name = path.to_string_lossy();
1333 match read(&name, OpenFlags::SQLITE_OPEN_READ_ONLY) {
1334 Err(StoreError::Sqlite(error))
1335 if matches!(
1336 error.sqlite_error_code(),
1337 Some(rusqlite::ErrorCode::ReadOnly | rusqlite::ErrorCode::CannotOpen)
1338 ) =>
1339 {
1340 let mut wal = path.as_os_str().to_owned();
1342 wal.push("-wal");
1343 let wal = PathBuf::from(wal);
1344 if std::fs::metadata(&wal).is_ok_and(|wal| wal.len() > 0) {
1345 return Err(StoreError::Unread(wal));
1346 }
1347 let uri = format!(
1348 "file:{}?mode=ro&immutable=1",
1349 name.replace('%', "%25")
1350 .replace('?', "%3f")
1351 .replace('#', "%23")
1352 );
1353 read(
1354 &uri,
1355 OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_URI,
1356 )
1357 }
1358 history => history,
1359 }
1360 .map(Some)
1361}
1362
1363fn sizing_of(db: &Connection, since: u64) -> Result<sizing::History, StoreError> {
1365 match version(db)? {
1366 0 => return Ok(sizing::History::default()),
1367 VERSION => {}
1368 _ => read_as_current(db)?,
1369 }
1370 let mut history = read::sizing(db, since)?;
1371 let auto = read::auto_sizing(db)?;
1372 history.memory_set = auto.memory;
1373 history.slots_set = auto.slots;
1374 history.changes = read::sizing_changes(db, since.saturating_sub(sizing::memory::HISTORY))?;
1375 Ok(history)
1376}
1377
1378fn version(db: &Connection) -> Result<u32, StoreError> {
1380 let found: u32 = db.pragma_query_value(None, "user_version", |row| row.get(0))?;
1381 match found {
1382 0 | OLDEST..=VERSION => Ok(found),
1383 found if found > VERSION => Err(StoreError::Newer {
1384 found,
1385 supported: VERSION,
1386 }),
1387 found => Err(StoreError::Older {
1388 found,
1389 oldest: OLDEST,
1390 }),
1391 }
1392}
1393
1394fn read_as_current(db: &Connection) -> Result<(), StoreError> {
1398 let current = Connection::open_in_memory()?;
1399 current.execute_batch(SCHEMA)?;
1400 let tables: Vec<(String, String)> = current
1401 .prepare(
1402 "SELECT name, sql FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'",
1403 )?
1404 .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?
1405 .collect::<rusqlite::Result<_>>()?;
1406 let columns =
1407 |db: &Connection, table: &str| -> rusqlite::Result<Vec<(String, Option<String>)>> {
1408 db.prepare("SELECT name, dflt_value FROM pragma_table_info(?1, 'main')")?
1409 .query_map([table], |row| Ok((row.get(0)?, row.get(1)?)))?
1410 .collect()
1411 };
1412 for (table, sql) in tables {
1413 let found: Vec<String> = columns(db, &table)?
1414 .into_iter()
1415 .map(|(name, _)| name)
1416 .collect();
1417 if found.is_empty() {
1418 db.execute_batch(&sql.replacen("CREATE TABLE", "CREATE TEMP TABLE", 1))?;
1419 continue;
1420 }
1421 let added: Vec<String> = columns(¤t, &table)?
1422 .into_iter()
1423 .filter(|(name, _)| !found.contains(name))
1424 .map(|(name, default)| format!("{} AS {name}", default.as_deref().unwrap_or("NULL")))
1425 .collect();
1426 if !added.is_empty() {
1427 db.execute_batch(&format!(
1428 "CREATE TEMP VIEW {table} AS SELECT *, {} FROM main.{table}",
1429 added.join(", ")
1430 ))?;
1431 }
1432 }
1433 Ok(())
1434}
1435
1436fn configure(db: &Connection) -> rusqlite::Result<()> {
1438 db.pragma_update(None, "journal_mode", "wal")?;
1439 db.pragma_update(None, "synchronous", "normal")?;
1440 db.pragma_update(None, "foreign_keys", "on")?;
1441 db.busy_timeout(Duration::from_secs(5))
1442}
1443
1444fn prepare(db: &mut Connection) -> Result<(), StoreError> {
1447 let found = version(db)?;
1448 if found == 0 {
1449 let transaction = db.transaction()?;
1450 transaction.execute_batch(SCHEMA)?;
1451 transaction.pragma_update(None, "user_version", VERSION)?;
1452 transaction.commit()?;
1453 return Ok(());
1454 }
1455 for (version, migration) in MIGRATIONS.iter().filter(|(version, _)| *version > found) {
1456 let transaction = db.transaction()?;
1457 transaction.execute_batch(migration)?;
1458 transaction.pragma_update(None, "user_version", version)?;
1459 transaction.commit()?;
1460 }
1461 Ok(())
1462}
1463
1464#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1469struct Downtime {
1470 from: u64,
1471 until: u64,
1472}
1473
1474impl Downtime {
1475 fn before(db: &mut Connection, now: u64) -> Result<Option<Downtime>, StoreError> {
1479 let transaction = db.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?;
1482 let last: Option<u64> = transaction.query_row(
1483 "SELECT max(at) FROM (SELECT max(at) AS at FROM host_samples UNION ALL SELECT max(at) FROM events \
1484 UNION ALL SELECT value FROM meta WHERE key = 'opened_at')",
1485 [],
1486 |row| row.get(0),
1487 )?;
1488 let downtime = last
1489 .filter(|last| *last < now)
1490 .map(|from| Downtime { from, until: now });
1491 if let Some(downtime) = downtime {
1492 let queued: Vec<(u64, u64)> = transaction
1493 .prepare("SELECT id, queued_at FROM jobs WHERE state = 'queued'")?
1494 .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?
1495 .collect::<rusqlite::Result<_>>()?;
1496 for (job, queued_at) in queued {
1497 transaction.execute(
1498 "UPDATE jobs SET unseen = unseen + ?2 WHERE id = ?1",
1499 rusqlite::params![job, downtime.of(queued_at)],
1500 )?;
1501 }
1502 }
1503 transaction.execute(
1504 "INSERT OR REPLACE INTO meta (key, value) VALUES ('opened_at', ?1)",
1505 [now],
1506 )?;
1507 transaction.commit()?;
1508 Ok(downtime)
1509 }
1510
1511 fn of(self, queued_at: u64) -> u64 {
1513 self.until.saturating_sub(queued_at.max(self.from))
1514 }
1515
1516 fn after(self, queued_at: u64, started_at: u64) -> u64 {
1519 self.of(queued_at)
1520 .saturating_sub(started_at.saturating_sub(queued_at.max(self.from)))
1521 }
1522}
1523
1524fn logged<T>(what: &str, result: rusqlite::Result<T>) -> Option<T> {
1526 result
1527 .map_err(|error| warn!("job history: cannot {what}: {error}"))
1528 .ok()
1529}
1530
1531#[cfg(test)]
1532mod tests;