use rusqlite::Connection;
use super::{keep_since, queued, runner, user_version};
use crate::api::{Bucket, HostSample, StatsQuery};
use crate::config::Config;
use crate::github::QueuedJob;
use crate::store::{SCHEMA, Store, VERSION};
use crate::testdir::TestDir;
const SCHEMA_15: &str = include_str!("schema-15.sql");
const SECOND: u64 = 1000;
const HOUR: u64 = 3600 * SECOND;
fn history_15(path: &std::path::Path) {
let db = Connection::open(path).unwrap();
db.execute_batch(SCHEMA_15).unwrap();
db.execute_batch(
"UPDATE history_kept SET jobs_since = 0;
INSERT INTO runners (name, repository, profile, slot, container, started_at, removed_at, memory_cap)
VALUES ('r', 'o/r', 'app', 1, 'c', 1000000, 1100000, 4294967296);
INSERT INTO jobs (id, repository, run_id, run_attempt, name, labels, state, conclusion, runner, queued_at,
started_at, finished_at, memory_peak)
VALUES (1, 'o/r', 10, 1, 'build', '[]', 'completed', 'success', 'r', 1000000, 1030000, 1100000, 1073741824);",
)
.unwrap();
db.pragma_update(None, "user_version", 15).unwrap();
}
#[tokio::test]
async fn a_history_at_15_reads_read_only_as_it_is_and_opens_at_16_with_its_rows_and_their_waits() {
let dir = TestDir::new("store-15");
let path = dir.0.join("runnerd.db");
history_15(&path);
let history = super::super::sizing_at(&path, 0).unwrap().unwrap();
let wait = history.jobs.first().and_then(crate::sizing::Ran::wait);
assert_eq!(
wait.map(|wait| wait.counted(history.kept_since)),
Some(30 * SECOND)
);
assert_eq!(
history.jobs.first().map(|job| job.slot_waited),
Some(0),
"a row from before schema 18 counts as no wait for a slot"
);
assert_eq!(user_version(&path), 15);
let store = Store::open(&path).unwrap();
assert_eq!(user_version(&path), VERSION);
let job = store.job(1).await.unwrap().unwrap().summary;
assert_eq!((job.timeline.unseen, job.waited(0)), (0, Some(30)));
let activity = store.activity(1000, 1100).await.unwrap();
assert_eq!(activity.wait_p95(), Some(30), "an old row keeps its wait");
}
fn described(db: &Connection) -> Vec<String> {
let mut tables: Vec<String> = db
.prepare("SELECT name FROM sqlite_master WHERE type = 'table' ORDER BY name")
.unwrap()
.query_map([], |row| row.get(0))
.unwrap()
.collect::<rusqlite::Result<_>>()
.unwrap();
tables.retain(|table| table != "sqlite_sequence");
let mut described = Vec::new();
for table in tables {
let columns = db
.prepare(
"SELECT cid, name, type, \"notnull\", dflt_value, pk FROM pragma_table_info(?1) ORDER BY cid",
)
.unwrap()
.query_map([&table], |row| {
Ok(format!(
"{table}: {} {} {} not null {} default {:?} pk {}",
row.get::<_, i64>(0)?,
row.get::<_, String>(1)?,
row.get::<_, String>(2)?,
row.get::<_, bool>(3)?,
row.get::<_, Option<String>>(4)?,
row.get::<_, i64>(5)?,
))
})
.unwrap()
.collect::<rusqlite::Result<Vec<_>>>()
.unwrap();
described.extend(columns);
}
let indexes = db
.prepare("SELECT tbl_name, name, sql FROM sqlite_master WHERE type = 'index' ORDER BY name")
.unwrap()
.query_map([], |row| {
Ok(format!(
"index {}.{}: {:?}",
row.get::<_, String>(0)?,
row.get::<_, String>(1)?,
row.get::<_, Option<String>>(2)?,
))
})
.unwrap()
.collect::<rusqlite::Result<Vec<_>>>()
.unwrap();
described.extend(indexes);
described
}
#[test]
fn a_history_brought_up_from_15_has_the_schema_a_new_one_gets() {
let dir = TestDir::new("store-drift");
let old = dir.0.join("old.db");
history_15(&old);
drop(Store::open(&old).unwrap());
let new = Connection::open_in_memory().unwrap();
new.execute_batch(SCHEMA).unwrap();
assert_eq!(described(&Connection::open(&old).unwrap()), described(&new));
}
fn rfc3339(at: u64) -> String {
Connection::open_in_memory()
.unwrap()
.query_row(
"SELECT strftime('%Y-%m-%dT%H:%M:%SZ', ?1, 'unixepoch')",
[at / SECOND],
|row| row.get(0),
)
.unwrap()
}
const CONFIG: &str = r#"
[host]
slice = "runnerd.slice"
isolation = "sysbox"
[caches]
mirror = { enabled = false, memory = "512M", cpus = 1, ttl = "168h" }
pnpm = { max_size = "5G" }
maven = { max_size = "5G" }
[profiles.app]
image_stage = "node"
cpus = 4
memory = "4G"
slots = 2
[[repositories]]
name = "o/r"
profile = "app"
labels = ["self-hosted"]
"#;
#[tokio::test]
async fn waits_leave_out_the_time_no_runnerd_ran_wherever_they_are_counted() {
let dir = TestDir::new("store-downtime");
let path = dir.0.join("runnerd.db");
let start = crate::time::now_ms() / SECOND * SECOND + SECOND;
let queue = |store: &Store, id: u64, at: u64| {
store.queued(
"o/r",
&QueuedJob {
created_at: at,
..queued(id)
},
crate::time::now_ms(),
);
};
let first = Store::open_at(&path, start).unwrap();
keep_since(&path, start);
queue(&first, 1, start + 10 * SECOND);
queue(&first, 5, start + 5 * SECOND);
first.host_sample(HostSample {
at: (start + 20 * SECOND) / SECOND,
cpu: 0,
memory: 0,
});
first.flush().await;
drop(first);
let restart = start + 8 * HOUR;
let second = Store::open_at(&path, restart).unwrap();
queue(&second, 2, start - HOUR);
queue(&second, 3, start + HOUR);
queue(&second, 4, restart + 60 * SECOND);
let started = restart + 100 * SECOND;
for job in 1..=5 {
second.claimed(job, 10, "r", started);
}
second.details(crate::github::JobDetails {
id: 5,
run_id: 10,
run_attempt: 1,
name: "job 5".to_owned(),
workflow_name: None,
head_branch: None,
html_url: None,
labels: Vec::new(),
created_at: None,
status: "in_progress".to_owned(),
conclusion: None,
started_at: Some(rfc3339(start + 2 * HOUR)),
completed_at: None,
runner_name: Some("r".to_owned()),
steps: Vec::new(),
});
for job in 1..=5 {
second.lost(job, started + 60 * SECOND);
}
second.flush().await;
let waits = [110, 120, 100, 40, 15];
let now = (started + HOUR) / SECOND;
let activity = second.activity(0, now).await.unwrap();
let mut counted: Vec<u64> = activity.started.iter().map(|job| job.waited).collect();
counted.sort_unstable();
let mut expected = waits.to_vec();
expected.sort_unstable();
assert_eq!(counted, expected, "the dashboard's waits");
let config = Config::parse(CONFIG).unwrap();
let history = second.insights(0).await.unwrap();
let insights = crate::insights::insights(&history, &config, 14, now * SECOND);
let week = &insights.waits.first().unwrap().now.all;
assert_eq!(
(week.jobs, week.p95),
(5, activity.wait_p95()),
"insights' p95 is the dashboard's"
);
let stats = second
.stats(
StatsQuery {
repository: None,
since: Some(0),
until: None,
bucket: Bucket::Day,
},
now,
)
.await
.unwrap();
let queued: u64 = stats.rows.iter().map(|row| row.queue_seconds).sum();
assert_eq!(queued, waits.iter().sum::<u64>(), "the stats' summed waits");
let job = second.job(3).await.unwrap().unwrap().summary;
assert_eq!(job.waited(now), Some(7 * 3600 + 100));
assert_eq!(
job.unseen_note(now).as_deref(),
Some("7h 00m of it while runnerd was not running")
);
let seen = second.job(4).await.unwrap().unwrap().summary;
assert_eq!((seen.waited(now), seen.unseen_note(now)), (Some(40), None));
}
#[tokio::test]
async fn an_open_whose_start_then_fails_does_not_hand_out_the_same_downtime_again() {
let dir = TestDir::new("store-reopen");
let path = dir.0.join("runnerd.db");
let start = crate::time::now_ms() / SECOND * SECOND + SECOND;
let first = Store::open_at(&path, start).unwrap();
first.queued(
"o/r",
&QueuedJob {
created_at: start + 10 * SECOND,
..queued(1)
},
crate::time::now_ms(),
);
first.flush().await;
drop(first);
let restart = start + 8 * HOUR;
drop(Store::open_at(&path, restart).unwrap());
let retried = Store::open_at(&path, restart + 3 * SECOND).unwrap();
drop(retried);
let store = Store::open_at(&path, restart + 3 * SECOND).unwrap();
let job = store.job(1).await.unwrap().unwrap().summary;
assert_eq!(
job.timeline.unseen,
(8 * HOUR - 10 * SECOND + 3 * SECOND) / SECOND
);
}
#[tokio::test]
async fn an_open_waits_for_a_writer_that_still_holds_the_history() {
let dir = TestDir::new("store-busy");
let path = dir.0.join("runnerd.db");
drop(Store::open(&path).unwrap());
let writer = Connection::open(&path).unwrap();
writer
.execute_batch(
"BEGIN IMMEDIATE; INSERT INTO host_samples (at, cpu, memory) VALUES (1000, 0, 0)",
)
.unwrap();
let release = std::thread::spawn(move || {
std::thread::sleep(std::time::Duration::from_millis(300));
writer.execute_batch("COMMIT").unwrap();
});
let opened = Store::open(&path);
release.join().unwrap();
assert!(opened.is_ok(), "the open waits: {:?}", opened.err());
}
#[tokio::test]
async fn a_jobs_causes_add_up_and_a_row_from_before_schema_26_counts_none() {
use crate::api::WaitCause;
let dir = TestDir::new("store-26");
let path = dir.0.join("runnerd.db");
history_15(&path);
let store = Store::open(&path).unwrap();
for id in [2, 3] {
store.queued("o/r", &queued(id), 1_000_000);
}
let wait = |job, cause, ms| crate::store::CauseWait { job, cause, ms };
store.causes_waited(vec![
wait(1, WaitCause::Host, 40 * SECOND),
wait(2, WaitCause::Host, 40 * SECOND),
]);
store.causes_waited(vec![
wait(2, WaitCause::Host, 5 * SECOND),
wait(2, WaitCause::NoPool, SECOND),
]);
store.flush().await;
Connection::open(&path)
.unwrap()
.execute_batch(
"UPDATE jobs SET state = 'completed', conclusion = 'success', started_at = 1100000, \
finished_at = 1200000 WHERE id IN (2, 3);
UPDATE jobs SET wait_causes = json_set(wait_causes, '$.later', 7000) WHERE id = 3;",
)
.unwrap();
let history = store.insights(0).await.unwrap();
let causes: Vec<_> = history
.runs
.iter()
.map(|run| (run.id, run.causes.clone()))
.collect();
assert_eq!(
causes,
[
(1, None),
(
2,
Some([(WaitCause::Host, 45 * SECOND), (WaitCause::NoPool, SECOND)].into())
),
(3, Some([(WaitCause::Other, 7 * SECOND)].into())),
]
);
}
#[tokio::test]
async fn only_the_downtime_a_jobs_first_listing_found_came_before_that_listing() {
let dir = TestDir::new("store-unlisted");
let path = dir.0.join("runnerd.db");
let start = crate::time::now_ms() / SECOND * SECOND + SECOND;
let queue = |store: &Store, id: u64, at: u64| {
store.queued(
"o/r",
&QueuedJob {
created_at: at,
..queued(id)
},
crate::time::now_ms(),
);
};
let first = Store::open_at(&path, start).unwrap();
keep_since(&path, start);
queue(&first, 1, start + 5 * SECOND);
first.host_sample(HostSample {
at: (start + 20 * SECOND) / SECOND,
cpu: 0,
memory: 0,
});
first.flush().await;
drop(first);
let second = Store::open_at(&path, start + HOUR).unwrap();
queue(&second, 2, start + 30 * SECOND);
second.flush().await;
let unseen: Vec<(u64, u64)> = Connection::open(&path)
.unwrap()
.prepare("SELECT unseen, unseen_unlisted FROM jobs ORDER BY id")
.unwrap()
.query_map([], |row| Ok((row.get(0)?, row.get(1)?)))
.unwrap()
.collect::<rusqlite::Result<_>>()
.unwrap();
let down = HOUR - 20 * SECOND;
assert_eq!(
unseen,
[(down, 0), (HOUR - 30 * SECOND, HOUR - 30 * SECOND)],
"job 1's downtime came after its listing; job 2's before"
);
}
#[tokio::test]
async fn a_job_a_capture_recorded_first_leaves_out_the_downtime_its_wait_went_through() {
let dir = TestDir::new("store-capture-first");
let path = dir.0.join("runnerd.db");
let start = crate::time::now_ms() / SECOND * SECOND + SECOND;
let first = Store::open_at(&path, start).unwrap();
keep_since(&path, start);
first.host_sample(HostSample {
at: (start + 20 * SECOND) / SECOND,
cpu: 0,
memory: 0,
});
first.flush().await;
drop(first);
let restart = start + HOUR;
let second = Store::open_at(&path, restart).unwrap();
second.runner_started(runner("r", restart));
let captured = restart + 5 * SECOND;
second.claimed(7, 10, "r", captured);
second.claimed(8, 10, "r", captured);
second.queued(
"o/r",
&QueuedJob {
created_at: start + 60 * SECOND,
..queued(8)
},
captured - SECOND,
);
for job in [7, 8] {
let details = crate::github::JobDetails {
id: job,
run_id: 10,
run_attempt: 1,
name: "build".to_owned(),
workflow_name: None,
head_branch: None,
html_url: None,
labels: Vec::new(),
created_at: Some(rfc3339(start + 60 * SECOND)),
status: "in_progress".to_owned(),
conclusion: None,
started_at: Some(rfc3339(start + 180 * SECOND)),
completed_at: None,
runner_name: Some("r".to_owned()),
steps: Vec::new(),
};
second.details(details.clone());
second.details(details);
}
second.flush().await;
let unseen: Vec<(u64, u64)> = Connection::open(&path)
.unwrap()
.prepare("SELECT unseen, unseen_unlisted FROM jobs ORDER BY id")
.unwrap()
.query_map([], |row| Ok((row.get(0)?, row.get(1)?)))
.unwrap()
.collect::<rusqlite::Result<_>>()
.unwrap();
let waited = 120 * SECOND;
assert_eq!(
unseen,
[(waited, waited), (waited, waited)],
"both waited out the downtime whole, before any listing"
);
}