1use std::collections::{BTreeMap, BTreeSet};
8use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt};
9use std::path::{Path, PathBuf};
10use std::sync::atomic::{AtomicBool, Ordering};
11use std::sync::{Arc, Mutex, TryLockError};
12use std::time::Duration;
13
14use rusqlite::{Connection, OpenFlags};
15use tokio::sync::{OwnedSemaphorePermit, Semaphore, broadcast, mpsc, oneshot};
16use tracing::warn;
17
18use crate::admission::trace::Trace;
19use crate::api::{self, Event, EventEnvelope};
20use crate::config::History;
21use crate::github::{JobDetails, NamedRun, QueuedJob};
22use crate::observe::disk::{Window, Written};
23use crate::observe::metrics::{CpuHeld, Events};
24use crate::scheduler::warm;
25use crate::sizing;
26use crate::sync::lock;
27
28mod lens;
29mod outline;
30mod prune;
31mod read;
32mod run;
33mod write;
34
35pub use run::{Claim, RunJobs, RunnerFault};
36
37const SCHEMA: &str = include_str!("store/schema.sql");
39
40const OLDEST: u32 = 15;
42
43const MIGRATIONS: &[(u32, &str)] = &[
46 (
47 16,
48 "ALTER TABLE jobs ADD COLUMN unseen INTEGER NOT NULL DEFAULT 0;",
49 ),
50 (
51 17,
52 "ALTER TABLE jobs ADD COLUMN cpu_periods INTEGER;\n\
53 ALTER TABLE jobs ADD COLUMN cpu_throttled INTEGER;\n\
54 ALTER TABLE jobs ADD COLUMN cpu_stalled INTEGER;\n\
55 ALTER TABLE samples ADD COLUMN cpu_periods INTEGER;\n\
56 ALTER TABLE samples ADD COLUMN cpu_throttled INTEGER;\n\
57 ALTER TABLE samples ADD COLUMN cpu_stalled INTEGER;",
58 ),
59 (
60 18,
61 "ALTER TABLE jobs ADD COLUMN slot_waited INTEGER NOT NULL DEFAULT 0;",
62 ),
63 (
64 19,
65 "ALTER TABLE jobs ADD COLUMN seen_at INTEGER;\n\
66 ALTER TABLE runners ADD COLUMN purpose TEXT;\n\
67 ALTER TABLE runners ADD COLUMN stopped_for TEXT;",
68 ),
69 (
70 20,
71 "ALTER TABLE jobs ADD COLUMN runner_fault INTEGER NOT NULL DEFAULT 0;",
72 ),
73 (21, "ALTER TABLE runners ADD COLUMN cancel_by INTEGER;"),
74 (
75 22,
76 "CREATE TABLE log_outlines (job INTEGER PRIMARY KEY NOT NULL, from_line INTEGER NOT NULL, \
77 errors INTEGER NOT NULL, warnings INTEGER NOT NULL, groups_kept INTEGER NOT NULL, \
78 marks_kept INTEGER NOT NULL, sections_kept INTEGER NOT NULL, dropped INTEGER NOT NULL, \
79 hidden INTEGER NOT NULL);\n\
80 CREATE TABLE log_groups (job INTEGER NOT NULL, start INTEGER NOT NULL, end_line INTEGER, \
81 depth INTEGER NOT NULL, title TEXT NOT NULL, PRIMARY KEY (job, start)) WITHOUT ROWID;\n\
82 CREATE INDEX log_groups_open ON log_groups (job, start) WHERE end_line IS NULL;\n\
83 CREATE TABLE log_marks (job INTEGER NOT NULL, line INTEGER NOT NULL, kind TEXT NOT NULL, \
84 PRIMARY KEY (job, line)) WITHOUT ROWID;",
85 ),
86 (
87 23,
88 "CREATE TABLE sessions (id INTEGER PRIMARY KEY, started_at INTEGER NOT NULL, sampled_at INTEGER);\n\
89 ALTER TABLE records ADD COLUMN gone_at INTEGER;",
90 ),
91 (
92 24,
93 "ALTER TABLE jobs ADD COLUMN disk_written INTEGER;\n\
94 ALTER TABLE jobs ADD COLUMN disk_write_ios INTEGER;\n\
95 ALTER TABLE jobs ADD COLUMN disk_dirty INTEGER;\n\
96 ALTER TABLE jobs ADD COLUMN disk_measure TEXT;\n\
97 ALTER TABLE runners ADD COLUMN cgroup_gone_at INTEGER;\n\
98 ALTER TABLE samples ADD COLUMN written INTEGER;\n\
99 ALTER TABLE samples ADD COLUMN write_ios INTEGER;\n\
100 ALTER TABLE samples ADD COLUMN dirty INTEGER;\n\
101 CREATE TABLE disk_windows (start INTEGER PRIMARY KEY NOT NULL, end_at INTEGER NOT NULL, \
102 device TEXT NOT NULL, store TEXT, drive INTEGER, slice INTEGER, daemon INTEGER, commits INTEGER) WITHOUT ROWID;\n\
103 CREATE INDEX jobs_by_runner ON jobs (runner);",
104 ),
105 (25, "ALTER TABLE jobs ADD COLUMN failure_cause TEXT;"),
106 (
107 26,
108 "ALTER TABLE jobs ADD COLUMN wait_causes TEXT;\n\
109 ALTER TABLE jobs ADD COLUMN unseen_unlisted INTEGER NOT NULL DEFAULT 0;",
110 ),
111 (
112 27,
113 "ALTER TABLE jobs ADD COLUMN rerun_at INTEGER;\n\
114 CREATE TABLE rerun_claims (repository TEXT NOT NULL, run_id INTEGER NOT NULL, attempt INTEGER NOT NULL, \
115 claimed_at INTEGER NOT NULL, PRIMARY KEY (repository, run_id, attempt)) WITHOUT ROWID;",
116 ),
117 (28, "ALTER TABLE runners ADD COLUMN probe TEXT;"),
118];
119
120const VERSION: u32 = match MIGRATIONS.last() {
122 Some((version, _)) => *version,
123 None => OLDEST,
124};
125
126pub(crate) const LOG_BACKLOG: u64 = 8 << 20;
129
130const HOST_SAMPLE_BATCH: usize = 30;
133
134const EVENT_BUFFER: usize = 4096;
136
137#[derive(Debug, thiserror::Error)]
139pub enum StoreError {
140 #[error("{0}")]
142 Sqlite(#[from] rusqlite::Error),
143 #[error(
145 "the job history database is version {found}, newer than the {supported} this runnerd reads; \
146 run a newer runnerd or move the database aside"
147 )]
148 Newer {
149 found: u32,
151 supported: u32,
153 },
154 #[error(
156 "the job history database is version {found}, from a runnerd before 0.0.19; this runnerd reads versions \
157 from {oldest} on: move the database aside"
158 )]
159 Older {
160 found: u32,
162 oldest: u32,
164 },
165 #[error("{0}")]
167 Io(#[from] std::io::Error),
168 #[error("a database read stopped: {0}")]
170 Join(#[from] tokio::task::JoinError),
171 #[error(
174 "{} holds changes not yet in the job history, which this user cannot read; run runnerd doctor as the user \
175 the daemon runs as",
176 .0.display()
177 )]
178 Unread(PathBuf),
179}
180
181impl StoreError {
182 pub fn is_shutdown(&self) -> bool {
185 matches!(self, StoreError::Join(join) if join.is_cancelled())
186 }
187}
188
189#[derive(Debug, Clone, Copy, PartialEq, Eq)]
191pub struct CauseWait {
192 pub job: u64,
194 pub cause: api::WaitCause,
196 pub ms: u64,
198}
199
200#[derive(Debug, Clone, PartialEq, Eq)]
202pub struct RunnerRecord {
203 pub name: String,
205 pub repository: String,
207 pub profile: String,
209 pub slot: usize,
211 pub container: String,
213 pub started_at: u64,
215 pub memory_cap: Option<u64>,
217 pub cpu_cap: Option<u64>,
219 pub admission: Option<api::Admission>,
221 pub purpose: Option<&'static str>,
223}
224
225#[derive(Debug, Clone, PartialEq, Eq)]
227pub struct SampleRecord {
228 pub runner: String,
230 pub at: u64,
232 pub cpu: u64,
234 pub cpu_usage: u64,
236 pub memory: u64,
238 pub network_received: u64,
240 pub network_sent: u64,
242 pub events: Events,
244 pub cpu_held: CpuHeld,
246 pub written: Option<Written>,
248 pub dirty: Option<u64>,
250}
251
252#[derive(Debug, Clone, Default, PartialEq, Eq)]
254pub struct Measured {
255 pub at: Option<u64>,
257 pub bytes: BTreeMap<String, u64>,
259}
260
261#[derive(Debug, Clone, Default, PartialEq, Eq)]
264pub struct Announced {
265 pub profiles: Vec<api::ProfileStatus>,
267 pub repositories: Vec<api::RepositoryStatus>,
269}
270
271#[derive(Debug, Clone, PartialEq, Eq)]
273pub struct OpenJob {
274 pub id: u64,
276 pub repository: String,
278 pub state: api::JobState,
280 pub runner: Option<String>,
282 pub runner_removed_at: Option<u64>,
284 pub run_id: u64,
286 pub run_attempt: u32,
288 pub workflow: Option<String>,
290 pub queued_at: u64,
292 pub labels: Vec<String>,
294}
295
296#[derive(Debug, Clone, PartialEq, Eq)]
298pub struct SlotJob {
299 pub profile: String,
301 pub slot: usize,
303 pub key: crate::admission::Key,
305}
306
307#[derive(Debug, Clone, Copy, PartialEq, Eq)]
309pub struct LogPosition {
310 pub page: u64,
312 pub offset: u64,
314}
315
316#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
320pub struct MirrorPosition {
321 pub at: u64,
323 pub seen: u64,
325}
326
327#[derive(Debug)]
329enum Write {
330 Queued {
331 repository: String,
332 job: QueuedJob,
333 seen_at: u64,
334 },
335 RunnerStarted(RunnerRecord),
336 RunnerAdmission {
337 runner: String,
338 admission: api::Admission,
339 },
340 RunnerOomKilled(String),
341 RunnerProbe {
342 runner: String,
343 probe: api::RunnerProbe,
344 },
345 RunnerStoppedFor {
346 runner: String,
347 reason: &'static str,
348 },
349 RunnerCancelBy {
350 runner: String,
351 at: u64,
352 },
353 HostWaited {
354 jobs: Vec<u64>,
355 ms: u64,
356 },
357 CausesWaited(Vec<CauseWait>),
358 SlotWaited {
359 jobs: Vec<u64>,
360 ms: u64,
361 },
362 Reran {
363 jobs: Vec<u64>,
364 at: u64,
365 },
366 BuildCache {
367 runner: String,
368 counts: api::BuildCacheCounts,
369 },
370 RunnerRemoved {
371 runner: String,
372 at: u64,
373 },
374 RunnersGone {
375 running: Vec<String>,
376 at: u64,
377 },
378 Claimed {
379 job: u64,
380 run_id: u64,
381 runner: String,
382 at: u64,
383 },
384 Details {
385 details: JobDetails,
386 },
387 WorkflowNamed(NamedRun),
388 Elsewhere {
389 details: JobDetails,
390 },
391 Lost {
392 job: u64,
393 at: u64,
394 },
395 Unanswered {
396 job: u64,
397 at: u64,
398 },
399 Log {
400 job: u64,
401 lines: Vec<String>,
402 position: Option<LogPosition>,
404 max_bytes: u64,
405 _backlog: OwnedSemaphorePermit,
407 },
408 Samples(Vec<SampleRecord>),
409 CgroupsGone {
410 runners: Vec<String>,
411 at: u64,
412 },
413 DiskWindow(Window),
414 HostSample(api::HostSample),
415 MirrorBytes {
416 bytes: Vec<(String, u64)>,
417 until: MirrorPosition,
418 done: oneshot::Sender<bool>,
420 },
421 CacheSizes {
422 at: u64,
423 sizes: BTreeMap<String, u64>,
424 },
425 Event(Box<Event>),
427 RecordMade(api::Record),
428 RecordSkipped {
429 job: u64,
430 why: String,
431 },
432 RecordRemoved {
433 job: u64,
434 at: u64,
435 by: api::RecordRemoval,
436 },
437 RecordsListed {
438 jobs: Vec<u64>,
439 at: u64,
440 },
441 Paused {
442 repositories: Vec<String>,
443 done: oneshot::Sender<bool>,
445 },
446 Visibility {
447 repository: String,
448 visibility: api::Visibility,
449 at: u64,
450 },
451 SizingIntent {
452 change: api::SizingChange,
453 done: oneshot::Sender<Option<i64>>,
455 },
456 SizingResolved {
457 id: i64,
458 resolution: Resolution,
459 done: oneshot::Sender<bool>,
461 },
462 Notify {
463 change: NotifyChange,
464 done: oneshot::Sender<bool>,
466 },
467 RunnerNamed {
468 secs: u64,
469 done: oneshot::Sender<bool>,
471 },
472 ClaimRerun {
473 attempt: run::ClaimKey,
474 at: u64,
475 done: oneshot::Sender<Claim>,
476 },
477 ReleaseRerun(run::ClaimKey),
478 Flush(oneshot::Sender<()>),
479}
480
481#[derive(Debug)]
483enum NotifyChange {
484 Record {
486 target: String,
487 set: Vec<(String, u64)>,
488 remove: Vec<String>,
489 },
490 Keep(Vec<String>),
492}
493
494#[derive(Debug, Clone, Copy, PartialEq, Eq)]
496pub enum Resolution {
497 Done,
499 Failed,
501 Uncertain,
504}
505
506#[derive(Debug, Clone, Default, PartialEq, Eq)]
508pub struct AutoSizing {
509 pub last: BTreeMap<String, u64>,
511 pub pending: Vec<(i64, api::SizingChange)>,
513 pub cpus: Vec<sizing::CpusSet>,
516 pub memory: Vec<sizing::MemorySet>,
518 pub slots: Vec<sizing::SlotsSet>,
521}
522
523#[derive(Debug, Clone)]
525pub struct Store {
526 writes: mpsc::UnboundedSender<Write>,
527 reader: Arc<Mutex<Connection>>,
528 pruner: Arc<Mutex<Connection>>,
530 closed: Arc<AtomicBool>,
532 held: Arc<Mutex<Vec<api::HostSample>>>,
535 latest_host: Arc<Mutex<Option<api::HostSample>>>,
537 latest_host_memory: Arc<Mutex<Option<api::HostMemory>>>,
539 host_waiting: Arc<Mutex<(BTreeSet<u64>, u64)>>,
541 latest_samples: Arc<Mutex<BTreeMap<String, SampleRecord>>>,
543 gone: Arc<Mutex<BTreeSet<String>>>,
545 events: broadcast::Sender<Arc<EventEnvelope>>,
546 backlog: Arc<Semaphore>,
547}
548
549impl Store {
550 pub fn open(path: &Path) -> Result<Store, StoreError> {
555 Store::open_at(path, crate::time::now_ms())
556 }
557
558 fn open_at(path: &Path, now: u64) -> Result<Store, StoreError> {
560 if let Some(dir) = path.parent().filter(|dir| !dir.as_os_str().is_empty()) {
561 std::fs::DirBuilder::new()
562 .recursive(true)
563 .mode(0o700)
564 .create(dir)?;
565 }
566 std::fs::OpenOptions::new()
568 .create(true)
569 .append(true)
570 .mode(0o600)
571 .open(path)?;
572 let mut db = Connection::open(path)?;
573 configure(&db)?;
574 prepare(&mut db)?;
575 let downtime = Downtime::before(&mut db, now)?;
576 db.execute("INSERT INTO sessions (started_at) VALUES (?1)", [now])?;
577 let session = db.last_insert_rowid();
578 let reader = Connection::open_with_flags(
579 path,
580 OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX,
581 )?;
582 configure(&reader)?;
583 let pruner = Connection::open(path)?;
584 configure(&pruner)?;
585 let (events, _) = broadcast::channel(EVENT_BUFFER);
586 let (writes, receiver) = mpsc::unbounded_channel();
587 let thread_events = events.clone();
588 let held = Arc::new(Mutex::new(Vec::new()));
589 let thread_held = held.clone();
590 std::thread::Builder::new()
591 .name("runnerd-store".to_owned())
592 .spawn(move || {
593 write::run(
594 write::Started {
595 db,
596 downtime,
597 session,
598 },
599 receiver,
600 &thread_events,
601 &thread_held,
602 );
603 })?;
604 Ok(Store {
605 writes,
606 reader: Arc::new(Mutex::new(reader)),
607 pruner: Arc::new(Mutex::new(pruner)),
608 closed: Arc::new(AtomicBool::new(false)),
609 held,
610 latest_host: Arc::new(Mutex::new(None)),
611 latest_host_memory: Arc::new(Mutex::new(None)),
612 host_waiting: Arc::new(Mutex::new((BTreeSet::new(), 0))),
613 latest_samples: Arc::new(Mutex::new(BTreeMap::new())),
614 gone: Arc::new(Mutex::new(BTreeSet::new())),
615 events,
616 backlog: Arc::new(Semaphore::new(
617 usize::try_from(LOG_BACKLOG).unwrap_or(usize::MAX),
618 )),
619 })
620 }
621
622 fn send(&self, write: Write) {
623 let _ = self.writes.send(write);
625 }
626
627 pub fn queued(&self, repository: &str, job: &QueuedJob, seen_at: u64) {
631 self.send(Write::Queued {
632 repository: repository.to_owned(),
633 job: job.clone(),
634 seen_at,
635 });
636 }
637
638 pub fn runner_started(&self, runner: RunnerRecord) {
640 self.send(Write::RunnerStarted(runner));
641 }
642
643 pub fn runner_removed(&self, runner: &str, at: u64) {
645 lock(&self.latest_samples).remove(runner);
646 lock(&self.gone).remove(runner);
647 self.send(Write::RunnerRemoved {
648 runner: runner.to_owned(),
649 at,
650 });
651 }
652
653 pub fn runners_gone(&self, running: Vec<String>, at: u64) {
656 lock(&self.latest_samples).retain(|runner, _| running.contains(runner));
657 lock(&self.gone).retain(|runner| running.contains(runner));
658 self.send(Write::RunnersGone { running, at });
659 }
660
661 pub fn claimed(&self, job: u64, run_id: u64, runner: &str, at: u64) {
663 self.send(Write::Claimed {
664 job,
665 run_id,
666 runner: runner.to_owned(),
667 at,
668 });
669 }
670
671 pub fn details(&self, details: JobDetails) {
673 self.send(Write::Details { details });
674 }
675
676 pub fn workflow_named(&self, named: NamedRun) {
679 self.send(Write::WorkflowNamed(named));
680 }
681
682 pub fn elsewhere(&self, details: JobDetails) {
685 self.send(Write::Elsewhere { details });
686 }
687
688 pub fn unanswered(&self, job: u64, at: u64) {
691 self.send(Write::Unanswered { job, at });
692 }
693
694 pub fn lost(&self, job: u64, at: u64) {
696 self.send(Write::Lost { job, at });
697 }
698
699 pub async fn log(&self, job: u64, lines: Vec<String>, position: LogPosition, max_bytes: u64) {
702 let size = |line: &String| line.len() as u64 + 1;
703 let mut lines = lines.into_iter().map(|mut line| {
704 if size(&line) > LOG_BACKLOG {
706 let mut end = usize::try_from(LOG_BACKLOG / 2).unwrap_or(usize::MAX);
707 while !line.is_char_boundary(end) {
708 end -= 1;
709 }
710 line.truncate(end);
711 }
712 line
713 });
714 let mut next = lines.next();
715 loop {
716 let (mut part, mut bytes) = (Vec::new(), 0);
717 while let Some(line) = next.take_if(|line| bytes + size(line) <= LOG_BACKLOG) {
718 bytes += size(&line);
719 part.push(line);
720 next = lines.next();
721 }
722 let last = next.is_none();
723 let share = u32::try_from(bytes.max(1)).unwrap_or(u32::MAX);
724 let Ok(backlog) = self.backlog.clone().acquire_many_owned(share).await else {
726 return;
727 };
728 self.send(Write::Log {
729 job,
730 lines: part,
731 position: last.then_some(position),
732 max_bytes,
733 _backlog: backlog,
734 });
735 if last {
736 return;
737 }
738 }
739 }
740
741 pub fn samples(&self, samples: Vec<SampleRecord>) {
743 let mut latest = lock(&self.latest_samples);
744 for sample in &samples {
745 latest.insert(sample.runner.clone(), sample.clone());
746 }
747 drop(latest);
748 self.send(Write::Samples(samples));
749 }
750
751 pub fn host_sample(&self, sample: api::HostSample) {
754 *lock(&self.latest_host) = Some(sample.clone());
755 lock(&self.held).push(sample.clone());
756 self.send(Write::HostSample(sample));
757 }
758
759 pub fn latest_sample(&self, runner: &str) -> Option<SampleRecord> {
761 lock(&self.latest_samples).get(runner).cloned()
762 }
763
764 pub fn containers_gone(&self, runners: Vec<String>) {
767 lock(&self.gone).extend(runners);
768 }
769
770 pub fn containers_ended(&self, runners: Vec<String>, at: u64) {
774 if !runners.is_empty() {
775 self.send(Write::CgroupsGone { runners, at });
776 }
777 }
778
779 pub fn disk_window(&self, window: Window) {
781 self.send(Write::DiskWindow(window));
782 }
783
784 pub fn containers_read(&self, runners: &[String]) {
787 let mut gone = lock(&self.gone);
788 for runner in runners {
789 gone.remove(runner);
790 }
791 }
792
793 pub fn container_gone(&self, runner: &str) -> bool {
795 lock(&self.gone).contains(runner)
796 }
797
798 pub fn runner_admission(&self, runner: &str, admission: api::Admission) {
800 self.send(Write::RunnerAdmission {
801 runner: runner.to_owned(),
802 admission,
803 });
804 }
805
806 pub fn runner_oom_killed(&self, runner: &str) {
809 self.send(Write::RunnerOomKilled(runner.to_owned()));
810 }
811
812 pub fn runner_probe(&self, runner: &str, probe: api::RunnerProbe) {
814 self.send(Write::RunnerProbe {
815 runner: runner.to_owned(),
816 probe,
817 });
818 }
819
820 pub fn runner_cancel_by(&self, runner: &str, at: u64) {
823 self.send(Write::RunnerCancelBy {
824 runner: runner.to_owned(),
825 at,
826 });
827 }
828
829 pub fn runner_stopped_for(&self, runner: &str, reason: &'static str) {
831 self.send(Write::RunnerStoppedFor {
832 runner: runner.to_owned(),
833 reason,
834 });
835 }
836
837 pub fn build_cache(&self, runner: &str, counts: api::BuildCacheCounts) {
839 self.send(Write::BuildCache {
840 runner: runner.to_owned(),
841 counts,
842 });
843 }
844
845 pub fn host_memory(&self, memory: api::HostMemory) {
847 *lock(&self.latest_host_memory) = Some(memory);
848 }
849
850 pub fn latest_host_memory(&self) -> Option<api::HostMemory> {
852 lock(&self.latest_host_memory).clone()
853 }
854
855 pub fn host_waiting(&self, jobs: BTreeSet<u64>, at: u64) {
858 *lock(&self.host_waiting) = (jobs, at);
859 }
860
861 pub fn host_waited(&self, jobs: Vec<u64>, ms: u64) {
864 self.send(Write::HostWaited { jobs, ms });
865 }
866
867 pub fn slot_waited(&self, jobs: Vec<u64>, ms: u64) {
870 self.send(Write::SlotWaited { jobs, ms });
871 }
872
873 pub fn causes_waited(&self, waits: Vec<CauseWait>) {
877 if waits.iter().any(|wait| wait.ms > 0) {
878 self.send(Write::CausesWaited(waits));
879 }
880 }
881
882 pub fn reran(&self, jobs: Vec<u64>, at: u64) {
885 self.send(Write::Reran { jobs, at });
886 }
887
888 pub async fn claim_rerun(&self, repository: &str, run: u64, attempt: u32, at: u64) -> Claim {
893 let (done, claimed) = oneshot::channel();
894 self.send(Write::ClaimRerun {
895 attempt: run::ClaimKey {
896 repository: repository.to_owned(),
897 run,
898 attempt,
899 },
900 at,
901 done,
902 });
903 tokio::task::spawn_blocking(move || claimed.blocking_recv().unwrap_or(Claim::Unstored))
905 .await
906 .unwrap_or(Claim::Unstored)
907 }
908
909 pub fn release_rerun(&self, repository: &str, run: u64, attempt: u32) {
912 self.send(Write::ReleaseRerun(run::ClaimKey {
913 repository: repository.to_owned(),
914 run,
915 attempt,
916 }));
917 }
918
919 pub fn latest_host_sample(&self) -> Option<api::HostSample> {
921 lock(&self.latest_host).clone()
922 }
923
924 pub async fn mirror_bytes(&self, bytes: Vec<(String, u64)>, until: MirrorPosition) -> bool {
927 let (done, committed) = oneshot::channel();
928 self.send(Write::MirrorBytes { bytes, until, done });
929 committed.await.unwrap_or(false)
930 }
931
932 pub fn cache_sizes(&self, at: u64, sizes: BTreeMap<String, u64>) {
934 self.send(Write::CacheSizes { at, sizes });
935 }
936
937 pub fn event(&self, event: Event) {
939 self.send(Write::Event(Box::new(event)));
940 }
941
942 pub fn record_made(&self, record: api::Record) {
944 self.send(Write::RecordMade(record));
945 }
946
947 pub fn record_skipped(&self, job: u64, why: String) {
949 self.send(Write::RecordSkipped { job, why });
950 }
951
952 pub fn record_removed(&self, job: u64, at: u64, by: api::RecordRemoval) {
954 self.send(Write::RecordRemoved { job, at, by });
955 }
956
957 pub fn records_listed(&self, jobs: Vec<u64>, at: u64) {
961 self.send(Write::RecordsListed { jobs, at });
962 }
963
964 pub fn paused(&self, repositories: Vec<String>) -> impl Future<Output = bool> + use<> {
967 let (done, committed) = oneshot::channel();
968 self.send(Write::Paused { repositories, done });
969 async move { committed.await.unwrap_or(false) }
970 }
971
972 pub async fn runner_named(&self, secs: u64) -> bool {
976 let (done, committed) = oneshot::channel();
977 self.send(Write::RunnerNamed { secs, done });
978 tokio::task::spawn_blocking(move || committed.blocking_recv().unwrap_or(false))
981 .await
982 .unwrap_or(false)
983 }
984
985 pub fn visibility(&self, repository: String, visibility: api::Visibility, at: u64) {
987 self.send(Write::Visibility {
988 repository,
989 visibility,
990 at,
991 });
992 }
993
994 pub fn sizing_intent(
997 &self,
998 change: api::SizingChange,
999 ) -> impl Future<Output = Option<i64>> + use<> {
1000 let (done, committed) = oneshot::channel();
1001 self.send(Write::SizingIntent { change, done });
1002 async move { committed.await.ok().flatten() }
1003 }
1004
1005 pub fn sizing_resolved(
1008 &self,
1009 id: i64,
1010 resolution: Resolution,
1011 ) -> impl Future<Output = bool> + use<> {
1012 let (done, committed) = oneshot::channel();
1013 self.send(Write::SizingResolved {
1014 id,
1015 resolution,
1016 done,
1017 });
1018 async move { committed.await.unwrap_or(false) }
1019 }
1020
1021 pub fn notify_record(
1024 &self,
1025 target: &str,
1026 set: Vec<(String, u64)>,
1027 remove: Vec<String>,
1028 ) -> impl Future<Output = bool> + use<> {
1029 let (done, committed) = oneshot::channel();
1030 self.send(Write::Notify {
1031 change: NotifyChange::Record {
1032 target: target.to_owned(),
1033 set,
1034 remove,
1035 },
1036 done,
1037 });
1038 async move { committed.await.unwrap_or(false) }
1039 }
1040
1041 pub fn notify_keep(&self, targets: Vec<String>) -> impl Future<Output = bool> + use<> {
1043 let (done, committed) = oneshot::channel();
1044 self.send(Write::Notify {
1045 change: NotifyChange::Keep(targets),
1046 done,
1047 });
1048 async move { committed.await.unwrap_or(false) }
1049 }
1050
1051 pub fn prune(&self, now: u64, history: &History) -> tokio::task::JoinHandle<()> {
1055 let pruner = self.pruner.clone();
1056 let closed = self.closed.clone();
1057 let history = history.clone();
1058 tokio::task::spawn_blocking(move || {
1059 let db = match pruner.try_lock() {
1060 Ok(db) => db,
1061 Err(TryLockError::Poisoned(poisoned)) => poisoned.into_inner(),
1062 Err(TryLockError::WouldBlock) => return,
1063 };
1064 let db = prune::Db {
1065 connection: &db,
1066 stop: &closed,
1067 };
1068 if !db.stop.load(Ordering::Relaxed) {
1069 prune::run(&db, now, &history);
1070 }
1071 })
1072 }
1073
1074 pub fn close(&self) {
1077 self.closed.store(true, Ordering::Relaxed);
1078 }
1079
1080 pub async fn flush(&self) {
1082 let (done, wait) = oneshot::channel();
1083 self.send(Write::Flush(done));
1084 let _ = wait.await;
1085 }
1086
1087 pub fn subscribe(&self) -> broadcast::Receiver<Arc<EventEnvelope>> {
1090 self.events.subscribe()
1091 }
1092
1093 async fn read<T: Send + 'static>(
1094 &self,
1095 query: impl FnOnce(&Connection) -> rusqlite::Result<T> + Send + 'static,
1096 ) -> Result<T, StoreError> {
1097 let reader = self.reader.clone();
1098 Ok(tokio::task::spawn_blocking(move || query(&lock(&reader))).await??)
1100 }
1101
1102 pub async fn jobs(&self, query: api::JobQuery) -> Result<api::JobList, StoreError> {
1104 self.read(move |db| read::jobs(db, &query)).await
1105 }
1106
1107 pub async fn timeline(&self, window: api::Span) -> Result<api::SlotTimeline, StoreError> {
1109 let held: Vec<u64> = lock(&self.held).iter().map(|sample| sample.at).collect();
1110 self.read(move |db| lens::timeline(db, window, &held, api::SAMPLE_INTERVAL))
1111 .await
1112 }
1113
1114 pub async fn compare(
1116 &self,
1117 id: u64,
1118 with: Option<u64>,
1119 ) -> Result<Option<api::JobComparison>, StoreError> {
1120 self.read(move |db| lens::compare(db, id, with)).await
1121 }
1122
1123 pub async fn job(&self, id: u64) -> Result<Option<api::Job>, StoreError> {
1125 self.read(move |db| read::job(db, id)).await
1126 }
1127
1128 pub async fn runner_faults(&self, since: u64) -> Result<Vec<RunnerFault>, StoreError> {
1131 self.read(move |db| run::runner_faults(db, since)).await
1132 }
1133
1134 pub async fn run_jobs(
1136 &self,
1137 repository: &str,
1138 run: u64,
1139 except: u64,
1140 limit: usize,
1141 ) -> Result<RunJobs, StoreError> {
1142 let repository = repository.to_owned();
1143 self.read(move |db| run::run_jobs(db, &repository, run, except, limit))
1144 .await
1145 }
1146
1147 pub async fn job_samples(
1149 &self,
1150 id: u64,
1151 interval: Duration,
1152 ) -> Result<Option<api::SampleSeries>, StoreError> {
1153 self.read(move |db| read::job_samples(db, id, interval))
1154 .await
1155 }
1156
1157 pub async fn log_lines(
1159 &self,
1160 job: u64,
1161 from: u64,
1162 limit: u64,
1163 ) -> Result<Vec<(u64, String)>, StoreError> {
1164 self.read(move |db| read::log_lines(db, job, from, limit))
1165 .await
1166 }
1167
1168 pub async fn log_outline(&self, job: u64) -> Result<Option<api::LogOutline>, StoreError> {
1170 self.read(move |db| outline::outline(db, job)).await
1171 }
1172
1173 pub async fn styled_lines(
1176 &self,
1177 job: u64,
1178 from: u64,
1179 limit: u64,
1180 ) -> Result<Vec<api::StyledLine>, StoreError> {
1181 self.read(move |db| outline::styled_lines(db, job, from, limit))
1182 .await
1183 }
1184
1185 pub async fn search_log(
1188 &self,
1189 job: u64,
1190 mut search: api::Search,
1191 ) -> Result<api::LogSearch, StoreError> {
1192 while let Some(want) = search.wants() {
1194 search = self
1195 .read(move |db| {
1196 search.read(read::lines_from(db, job, want)?);
1197 Ok(search)
1198 })
1199 .await?;
1200 }
1201 Ok(search.finish())
1202 }
1203
1204 pub async fn events_after(
1206 &self,
1207 after: u64,
1208 limit: u64,
1209 ) -> Result<Vec<EventEnvelope>, StoreError> {
1210 self.read(move |db| read::events_after(db, after, limit))
1211 .await
1212 }
1213
1214 pub async fn sizing_changes(&self, since: u64) -> Result<Vec<api::SizingChange>, StoreError> {
1216 self.read(move |db| read::sizing_changes(db, since)).await
1217 }
1218
1219 pub async fn auto_sizing(&self) -> Result<AutoSizing, StoreError> {
1221 self.read(read::auto_sizing).await
1222 }
1223
1224 pub fn announced(&self) -> Result<Announced, StoreError> {
1227 Ok(read::announced(&lock(&self.reader))?)
1228 }
1229
1230 pub fn paused_repositories(&self) -> Result<Vec<String>, StoreError> {
1232 Ok(read::paused(&lock(&self.reader))?)
1233 }
1234
1235 pub async fn host_samples(&self, since: u64) -> Result<Vec<api::HostSample>, StoreError> {
1237 let held = lock(&self.held).clone();
1240 let mut samples = self.read(move |db| read::host_samples(db, since)).await?;
1241 let stored = samples.last().map_or(0, |sample| sample.at);
1242 samples.extend(
1243 held.into_iter()
1244 .filter(|sample| sample.at >= since && sample.at > stored),
1245 );
1246 Ok(samples)
1247 }
1248
1249 pub async fn activity(&self, since: u64, now: u64) -> Result<api::Activity, StoreError> {
1251 self.read(move |db| read::activity(db, since, now)).await
1252 }
1253
1254 pub async fn stats(&self, query: api::StatsQuery, now: u64) -> Result<api::Stats, StoreError> {
1256 self.read(move |db| read::stats(db, &query, now)).await
1257 }
1258
1259 pub async fn sizing(&self, since: u64) -> Result<sizing::History, StoreError> {
1261 self.read(move |db| read::sizing(db, since)).await
1262 }
1263
1264 pub async fn pricing(&self, since: u64) -> Result<api::PricingHistory, StoreError> {
1267 self.read(move |db| read::pricing(db, since)).await
1268 }
1269
1270 pub async fn insights(&self, since: u64) -> Result<crate::insights::History, StoreError> {
1273 let memory = self.latest_host_memory();
1274 let (waiting, seen) = lock(&self.host_waiting).clone();
1275 self.read(move |db| {
1276 let mut history = read::insights(db, since)?;
1277 if let Some(memory) = memory.filter(|_| !waiting.is_empty()) {
1278 let jobs = read::held_jobs(db, &waiting)?;
1279 history.host =
1280 (!jobs.is_empty()).then_some(crate::insights::HostHold { memory, seen, jobs });
1281 }
1282 Ok(history)
1283 })
1284 .await
1285 }
1286
1287 pub async fn last_event(&self) -> Result<u64, StoreError> {
1289 self.read(read::last_event).await
1290 }
1291
1292 pub async fn notify_state(
1294 &self,
1295 ) -> Result<BTreeMap<String, BTreeMap<String, u64>>, StoreError> {
1296 self.read(read::notify_state).await
1297 }
1298
1299 pub async fn warm(&self, since: u64) -> Result<Vec<warm::Seen>, StoreError> {
1301 self.read(move |db| read::warm(db, since)).await
1302 }
1303
1304 pub async fn cache_sizes_now(&self) -> Result<Measured, StoreError> {
1306 self.read(read::latest_cache_sizes).await
1307 }
1308
1309 pub async fn build_cache_since(&self, since: u64) -> Result<api::BuildCacheCounts, StoreError> {
1311 self.read(move |db| read::build_cache_since(db, since))
1312 .await
1313 }
1314
1315 pub async fn cache_history(&self, since: u64) -> Result<Vec<api::CacheSize>, StoreError> {
1317 self.read(move |db| read::cache_history(db, since)).await
1318 }
1319
1320 pub async fn open_jobs(&self) -> Result<Vec<OpenJob>, StoreError> {
1322 self.read(read::open_jobs).await
1323 }
1324
1325 pub async fn slot_jobs(&self) -> Result<Vec<SlotJob>, StoreError> {
1327 self.read(read::slot_jobs).await
1328 }
1329
1330 pub async fn last_runner_second(&self, prefix: String) -> Result<Option<u64>, StoreError> {
1334 self.read(move |db| read::last_runner_second(db, &prefix))
1335 .await
1336 }
1337
1338 pub async fn runner_job(&self, runner: &str) -> Result<Option<u64>, StoreError> {
1340 let runner = runner.to_owned();
1341 self.read(move |db| read::runner_job(db, &runner)).await
1342 }
1343
1344 pub async fn runner_admitted(
1346 &self,
1347 runner: &str,
1348 ) -> Result<Option<api::Admission>, StoreError> {
1349 let runner = runner.to_owned();
1350 self.read(move |db| read::runner_admission(db, &runner))
1351 .await
1352 }
1353
1354 pub async fn runner_cancel_deadline(&self, runner: &str) -> Result<Option<u64>, StoreError> {
1357 let runner = runner.to_owned();
1358 self.read(move |db| read::runner_cancel_deadline(db, &runner))
1359 .await
1360 }
1361
1362 pub async fn runner_memory_cap(&self, runner: &str) -> Result<Option<u64>, StoreError> {
1364 let runner = runner.to_owned();
1365 self.read(move |db| read::runner_memory_cap(db, &runner))
1366 .await
1367 }
1368
1369 pub async fn log_position(&self, job: u64) -> Result<LogPosition, StoreError> {
1371 self.read(move |db| read::log_position(db, job)).await
1372 }
1373
1374 pub async fn mirror_log_until(&self) -> Result<MirrorPosition, StoreError> {
1376 self.read(read::mirror_log_until).await
1377 }
1378}
1379
1380pub fn sizing_at(path: &Path, since: u64) -> Result<Option<sizing::History>, StoreError> {
1385 read_only(path, |db| sizing_of(db, since))
1386}
1387
1388pub fn trace_at(
1393 path: &Path,
1394 since: u64,
1395 named_by: u64,
1396 overhead: Option<u64>,
1397) -> Result<Option<Trace>, StoreError> {
1398 read_only(path, |db| {
1399 if version(db)? == 0 {
1400 return Ok(Trace::default());
1401 }
1402 Ok(read::trace(db, since, named_by, overhead)?)
1403 })
1404}
1405
1406pub fn last_queued_at(path: &Path) -> Result<Option<u64>, StoreError> {
1409 let last = read_only(path, |db| {
1410 if version(db)? == 0 {
1411 return Ok(None);
1412 }
1413 Ok(db.query_row("SELECT MAX(queued_at) FROM jobs", [], |row| row.get(0))?)
1414 })?;
1415 Ok(last.flatten())
1416}
1417
1418fn read_only<T>(
1420 path: &Path,
1421 read: impl Fn(&Connection) -> Result<T, StoreError>,
1422) -> Result<Option<T>, StoreError> {
1423 if !path.exists() {
1424 return Ok(None);
1425 }
1426 let read = |path: &str, flags| -> Result<T, StoreError> {
1427 let db = Connection::open_with_flags(path, flags)?;
1428 db.busy_timeout(Duration::from_secs(5))?;
1429 read(&db)
1430 };
1431 let name = path.to_string_lossy();
1432 match read(&name, OpenFlags::SQLITE_OPEN_READ_ONLY) {
1433 Err(StoreError::Sqlite(error))
1434 if matches!(
1435 error.sqlite_error_code(),
1436 Some(rusqlite::ErrorCode::ReadOnly | rusqlite::ErrorCode::CannotOpen)
1437 ) =>
1438 {
1439 let mut wal = path.as_os_str().to_owned();
1441 wal.push("-wal");
1442 let wal = PathBuf::from(wal);
1443 if std::fs::metadata(&wal).is_ok_and(|wal| wal.len() > 0) {
1444 return Err(StoreError::Unread(wal));
1445 }
1446 let uri = format!(
1447 "file:{}?mode=ro&immutable=1",
1448 name.replace('%', "%25")
1449 .replace('?', "%3f")
1450 .replace('#', "%23")
1451 );
1452 read(
1453 &uri,
1454 OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_URI,
1455 )
1456 }
1457 history => history,
1458 }
1459 .map(Some)
1460}
1461
1462fn sizing_of(db: &Connection, since: u64) -> Result<sizing::History, StoreError> {
1464 match version(db)? {
1465 0 => return Ok(sizing::History::default()),
1466 VERSION => {}
1467 _ => read_as_current(db)?,
1468 }
1469 let mut history = read::sizing(db, since)?;
1470 let auto = read::auto_sizing(db)?;
1471 history.memory_set = auto.memory;
1472 history.slots_set = auto.slots;
1473 history.changes = read::sizing_changes(db, since.saturating_sub(sizing::memory::HISTORY))?;
1474 Ok(history)
1475}
1476
1477fn version(db: &Connection) -> Result<u32, StoreError> {
1479 let found: u32 = db.pragma_query_value(None, "user_version", |row| row.get(0))?;
1480 match found {
1481 0 | OLDEST..=VERSION => Ok(found),
1482 found if found > VERSION => Err(StoreError::Newer {
1483 found,
1484 supported: VERSION,
1485 }),
1486 found => Err(StoreError::Older {
1487 found,
1488 oldest: OLDEST,
1489 }),
1490 }
1491}
1492
1493fn read_as_current(db: &Connection) -> Result<(), StoreError> {
1497 let current = Connection::open_in_memory()?;
1498 current.execute_batch(SCHEMA)?;
1499 let tables: Vec<(String, String)> = current
1500 .prepare(
1501 "SELECT name, sql FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'",
1502 )?
1503 .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?
1504 .collect::<rusqlite::Result<_>>()?;
1505 let columns =
1506 |db: &Connection, table: &str| -> rusqlite::Result<Vec<(String, Option<String>)>> {
1507 db.prepare("SELECT name, dflt_value FROM pragma_table_info(?1, 'main')")?
1508 .query_map([table], |row| Ok((row.get(0)?, row.get(1)?)))?
1509 .collect()
1510 };
1511 for (table, sql) in tables {
1512 let found: Vec<String> = columns(db, &table)?
1513 .into_iter()
1514 .map(|(name, _)| name)
1515 .collect();
1516 if found.is_empty() {
1517 db.execute_batch(&sql.replacen("CREATE TABLE", "CREATE TEMP TABLE", 1))?;
1518 continue;
1519 }
1520 let added: Vec<String> = columns(¤t, &table)?
1521 .into_iter()
1522 .filter(|(name, _)| !found.contains(name))
1523 .map(|(name, default)| format!("{} AS {name}", default.as_deref().unwrap_or("NULL")))
1524 .collect();
1525 if !added.is_empty() {
1526 db.execute_batch(&format!(
1527 "CREATE TEMP VIEW {table} AS SELECT *, {} FROM main.{table}",
1528 added.join(", ")
1529 ))?;
1530 }
1531 }
1532 Ok(())
1533}
1534
1535fn configure(db: &Connection) -> rusqlite::Result<()> {
1537 db.pragma_update(None, "journal_mode", "wal")?;
1538 db.pragma_update(None, "synchronous", "normal")?;
1539 db.pragma_update(None, "foreign_keys", "on")?;
1540 db.busy_timeout(Duration::from_secs(5))
1541}
1542
1543fn prepare(db: &mut Connection) -> Result<(), StoreError> {
1546 let found = version(db)?;
1547 if found == 0 {
1548 let transaction = db.transaction()?;
1549 transaction.execute_batch(SCHEMA)?;
1550 transaction.pragma_update(None, "user_version", VERSION)?;
1551 transaction.commit()?;
1552 return Ok(());
1553 }
1554 for (version, migration) in MIGRATIONS.iter().filter(|(version, _)| *version > found) {
1555 let transaction = db.transaction()?;
1556 transaction.execute_batch(migration)?;
1557 transaction.pragma_update(None, "user_version", version)?;
1558 transaction.commit()?;
1559 }
1560 Ok(())
1561}
1562
1563#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1568struct Downtime {
1569 from: u64,
1570 until: u64,
1571}
1572
1573impl Downtime {
1574 fn before(db: &mut Connection, now: u64) -> Result<Option<Downtime>, StoreError> {
1578 let transaction = db.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?;
1581 let last: Option<u64> = transaction.query_row(
1582 "SELECT max(at) FROM (SELECT max(at) AS at FROM host_samples UNION ALL SELECT max(at) FROM events \
1583 UNION ALL SELECT value FROM meta WHERE key = 'opened_at')",
1584 [],
1585 |row| row.get(0),
1586 )?;
1587 let downtime = last
1588 .filter(|last| *last < now)
1589 .map(|from| Downtime { from, until: now });
1590 if let Some(downtime) = downtime {
1591 let queued: Vec<(u64, u64)> = transaction
1592 .prepare("SELECT id, queued_at FROM jobs WHERE state = 'queued'")?
1593 .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?
1594 .collect::<rusqlite::Result<_>>()?;
1595 for (job, queued_at) in queued {
1596 transaction.execute(
1597 "UPDATE jobs SET unseen = unseen + ?2 WHERE id = ?1",
1598 rusqlite::params![job, downtime.of(queued_at)],
1599 )?;
1600 }
1601 }
1602 transaction.execute(
1603 "INSERT OR REPLACE INTO meta (key, value) VALUES ('opened_at', ?1)",
1604 [now],
1605 )?;
1606 transaction.commit()?;
1607 Ok(downtime)
1608 }
1609
1610 fn of(self, queued_at: u64) -> u64 {
1612 self.until.saturating_sub(queued_at.max(self.from))
1613 }
1614
1615 fn during(self, from: u64, until: u64) -> u64 {
1617 until.min(self.until).saturating_sub(from.max(self.from))
1618 }
1619
1620 fn after(self, queued_at: u64, started_at: u64) -> u64 {
1623 self.of(queued_at)
1624 .saturating_sub(started_at.saturating_sub(queued_at.max(self.from)))
1625 }
1626}
1627
1628fn logged<T>(what: &str, result: rusqlite::Result<T>) -> Option<T> {
1630 result
1631 .map_err(|error| warn!("job history: cannot {what}: {error}"))
1632 .ok()
1633}
1634
1635#[cfg(test)]
1636mod tests;