use std::fmt;
use std::num::NonZeroU16;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Mutex, MutexGuard, PoisonError};
use std::time::Duration;
use rusqlite::types::{Value, ValueRef};
use rusqlite::{Connection, OptionalExtension, Row, ToSql, TransactionBehavior, named_params};
use serde::Serialize;
use serde::de::DeserializeOwned;
use uuid::Uuid;
use crate::attempt::{AttemptError, AttemptOutcome, AttemptState, PersistedAttempt, RunnerAttempt};
use crate::model::{
Arch, AttemptId, CachePolicy, Clock, Host, HostId, HostLabel, Os, PolicyId, RefreshInterval,
ScaleTarget, StartMode, SystemClock, TargetScope, Timestamp, ValidationError,
};
use crate::path::LocalAbsolutePath;
use crate::policy::{PersistedPolicy, PolicyError, PolicyState, RoutingLabels, ScalePolicy};
use crate::workspace::{WorkspaceError, WorkspaceKind};
#[derive(Debug, thiserror::Error)]
pub enum StoreError {
#[error("the database at {path} could not be opened: {source}")]
Open {
path: PathBuf,
#[source]
source: rusqlite::Error,
},
#[error(transparent)]
Sqlite(#[from] rusqlite::Error),
#[error(
"this database is at schema version {found}, but this build of \
runner-manager understands version {supported}; upgrade runner-manager \
rather than running it against a database from a newer version"
)]
SchemaTooNew { found: u32, supported: u32 },
#[error("schema migration {version} ({name}) failed and was rolled back: {source}")]
Migration {
version: u32,
name: &'static str,
#[source]
source: rusqlite::Error,
},
#[error(
"policy {id} was written against revision {expected}, but the stored \
revision is now {found}; another process changed it first and nothing \
was written"
)]
StaleRevision {
id: PolicyId,
expected: u64,
found: u64,
},
#[error(
"policy {id} was confirmed with {expected} active runner(s), but now has \
{found}; nothing was written"
)]
ActiveCountChanged {
id: PolicyId,
expected: u16,
found: u16,
},
#[error(
"host {id} was written against runner root {expected}, but the stored \
override is now {found}; another process changed it first and nothing \
was written"
)]
RunnerRootChanged {
id: HostId,
expected: String,
found: String,
},
#[error(
"{subject} was confirmed with {expected} uncleaned attempt(s), but now \
has {found}; nothing was written"
)]
UncleanedCountChanged {
subject: String,
expected: u16,
found: u16,
},
#[error(
"policy {policy} already holds an uncleaned attempt in persistent slot \
s{slot}; one slot is leased to at most one uncleaned attempt and \
nothing was written"
)]
SlotAlreadyLeased { policy: PolicyId, slot: u16 },
#[error("no {what} with id {id} is in the database")]
NotFound { what: &'static str, id: String },
#[error("a {what} with id {id} is already in the database")]
AlreadyExists { what: &'static str, id: String },
#[error("the stored policy {id} is not a legal policy: {source}")]
CorruptPolicy {
id: PolicyId,
#[source]
source: PolicyError,
},
#[error("the stored attempt {id} is not a legal attempt: {source}")]
CorruptAttempt {
id: AttemptId,
#[source]
source: AttemptError,
},
#[error("the stored host {id} is not a legal host: {source}")]
CorruptHost {
id: HostId,
#[source]
source: ValidationError,
},
#[error("the stored host {id} has an unusable configured runner root: {source}")]
CorruptHostWorkspace {
id: HostId,
#[source]
source: WorkspaceError,
},
#[error("{table}.{column} of row {id} holds {value}, which is not {expected}")]
CorruptColumn {
table: &'static str,
column: &'static str,
id: String,
value: String,
expected: &'static str,
},
#[error(
"{what} is {value}, which does not fit in a SQLite integer; SQLite \
integers are signed 64-bit and this store will not silently truncate one"
)]
UnrepresentableInteger { what: &'static str, value: u64 },
#[error("attempt {attempt} has a runtime path that is not valid UTF-8: {path:?}")]
UnrepresentablePath { attempt: AttemptId, path: PathBuf },
}
impl StoreError {
#[must_use]
pub const fn is_conflict(&self) -> bool {
matches!(
self,
StoreError::StaleRevision { .. }
| StoreError::ActiveCountChanged { .. }
| StoreError::RunnerRootChanged { .. }
| StoreError::UncleanedCountChanged { .. }
| StoreError::SlotAlreadyLeased { .. }
)
}
}
#[derive(Debug, Clone, Copy)]
struct Migration {
version: u32,
name: &'static str,
sql: &'static str,
}
const MIGRATIONS: &[Migration] = &[
Migration {
version: 1,
name: "initial_schema",
sql: include_str!("store/migrations/0001_initial_schema.sql"),
},
Migration {
version: 2,
name: "policy_host_label",
sql: include_str!("store/migrations/0002_policy_host_label.sql"),
},
Migration {
version: 3,
name: "workspace_locations",
sql: include_str!("store/migrations/0003_workspace_locations.sql"),
},
];
pub const SCHEMA_VERSION: u32 = 3;
const BOOTSTRAP_SQL: &str = "\
CREATE TABLE IF NOT EXISTS schema_migrations (
version INTEGER NOT NULL PRIMARY KEY,
name TEXT NOT NULL,
applied_at TEXT NOT NULL
) STRICT;";
const TABLES: &[&str] = &["schema_migrations", "hosts", "policies", "attempts"];
fn current_version(conn: &Connection) -> Result<u32, StoreError> {
let max: Option<i64> =
conn.query_row("SELECT MAX(version) FROM schema_migrations", [], |row| {
row.get(0)
})?;
match max {
None => Ok(0),
Some(raw) => u32::try_from(raw).map_err(|_| StoreError::CorruptColumn {
table: "schema_migrations",
column: "version",
id: raw.to_string(),
value: clip(&raw.to_string()),
expected: "a schema version this build could have written",
}),
}
}
fn apply_migrations(
conn: &mut Connection,
migrations: &[Migration],
clock: &dyn Clock,
) -> Result<u32, StoreError> {
conn.execute_batch(BOOTSTRAP_SQL)?;
let supported = migrations.last().map_or(0, |m| m.version);
let found = current_version(conn)?;
if found > supported {
return Err(StoreError::SchemaTooNew { found, supported });
}
for migration in migrations.iter().filter(|m| m.version > found) {
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
let record = |source| StoreError::Migration {
version: migration.version,
name: migration.name,
source,
};
tx.execute_batch(migration.sql).map_err(record)?;
tx.execute(
"INSERT INTO schema_migrations (version, name, applied_at) \
VALUES (:version, :name, :applied_at)",
named_params! {
":version": i64::from(migration.version),
":name": migration.name,
":applied_at": timestamp_to_text(clock.now()),
},
)
.map_err(record)?;
tx.commit()?;
}
Ok(supported)
}
pub trait Store: fmt::Debug + Send + Sync {
fn put_host(&self, host: &Host) -> Result<(), StoreError>;
fn host(&self, id: HostId) -> Result<Option<Host>, StoreError>;
fn hosts(&self) -> Result<Vec<Host>, StoreError>;
fn set_runner_root_override(
&self,
id: HostId,
expected: Option<&LocalAbsolutePath>,
new_root: Option<&LocalAbsolutePath>,
expected_uncleaned: u16,
) -> Result<(), StoreError>;
fn insert_policy(&self, policy: &ScalePolicy) -> Result<(), StoreError>;
fn update_policy(&self, policy: &ScalePolicy, expected_revision: u64)
-> Result<(), StoreError>;
fn update_policy_confirming_active_count(
&self,
policy: &ScalePolicy,
expected_revision: u64,
expected_active: u16,
) -> Result<(), StoreError>;
fn update_policy_confirming_uncleaned_count(
&self,
policy: &ScalePolicy,
expected_revision: u64,
expected_uncleaned: u16,
) -> Result<(), StoreError>;
fn remove_policy(&self, id: PolicyId, expected_revision: u64) -> Result<(), StoreError>;
fn policy(&self, id: PolicyId) -> Result<Option<ScalePolicy>, StoreError>;
fn policies(&self) -> Result<Vec<ScalePolicy>, StoreError>;
fn record_attempt(&self, attempt: &RunnerAttempt) -> Result<(), StoreError>;
fn attempt(&self, id: AttemptId) -> Result<Option<RunnerAttempt>, StoreError>;
fn attempts(&self) -> Result<Vec<RunnerAttempt>, StoreError>;
fn attempts_for_policy(&self, policy_id: PolicyId) -> Result<Vec<RunnerAttempt>, StoreError>;
fn active_attempts_for_policy(
&self,
policy_id: PolicyId,
) -> Result<Vec<RunnerAttempt>, StoreError>;
fn uncleaned_attempts_for_policy(
&self,
policy_id: PolicyId,
) -> Result<Vec<RunnerAttempt>, StoreError>;
fn slot_leases_for_policy(&self, policy_id: PolicyId)
-> Result<Vec<RunnerAttempt>, StoreError>;
fn uncleaned_ephemeral_attempts(&self) -> Result<Vec<RunnerAttempt>, StoreError>;
fn remove_attempt(&self, id: AttemptId) -> Result<bool, StoreError>;
}
fn active_sql() -> String {
let states = AttemptState::ALL
.into_iter()
.filter(|state| state.counts_against_capacity())
.map(|state| format!("'{}'", token(&state)))
.collect::<Vec<_>>()
.join(", ");
format!("state IN ({states})")
}
fn uncleaned_sql() -> String {
format!("state <> '{}'", token(&AttemptState::Cleaned))
}
fn uncleaned_of_kind_sql(kind: WorkspaceKind) -> String {
format!(
"workspace_mode = '{}' AND {}",
token(&kind),
uncleaned_sql()
)
}
const ATTEMPT_COUNT_CEILING: u16 = u16::MAX;
fn attempt_count_sql(predicate: &str) -> String {
format!("SELECT MIN(COUNT(*), {ATTEMPT_COUNT_CEILING}) FROM attempts WHERE {predicate}")
}
fn clamped_count(found: i64) -> u16 {
u16::try_from(found).unwrap_or(ATTEMPT_COUNT_CEILING)
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum CountedAttempts {
Active,
Uncleaned,
}
impl CountedAttempts {
fn count_sql(self) -> String {
let predicate = match self {
CountedAttempts::Active => active_sql(),
CountedAttempts::Uncleaned => uncleaned_sql(),
};
attempt_count_sql(&format!("policy_id = :id AND {predicate}"))
}
fn count_changed(self, id: PolicyId, expected: u16, found: u16) -> StoreError {
match self {
CountedAttempts::Active => StoreError::ActiveCountChanged {
id,
expected,
found,
},
CountedAttempts::Uncleaned => StoreError::UncleanedCountChanged {
subject: format!("policy {id}"),
expected,
found,
},
}
}
}
pub struct SqliteStore {
conn: Mutex<Connection>,
path: Option<PathBuf>,
schema_version: u32,
journal_mode: String,
clock_skew_repairs: AtomicU64,
}
impl fmt::Debug for SqliteStore {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("SqliteStore")
.field("path", &self.path)
.field("schema_version", &self.schema_version)
.field("journal_mode", &self.journal_mode)
.field(
"clock_skew_repairs",
&self.clock_skew_repairs.load(Ordering::Relaxed),
)
.finish()
}
}
impl SqliteStore {
pub fn open(path: impl AsRef<Path>) -> Result<Self, StoreError> {
let path = path.as_ref();
let conn = Connection::open(path).map_err(|source| StoreError::Open {
path: path.to_path_buf(),
source,
})?;
Self::with_migrations(conn, Some(path.to_path_buf()), MIGRATIONS)
}
pub fn open_in_memory() -> Result<Self, StoreError> {
let conn = Connection::open_in_memory().map_err(|source| StoreError::Open {
path: PathBuf::from(":memory:"),
source,
})?;
Self::with_migrations(conn, None, MIGRATIONS)
}
fn with_migrations(
mut conn: Connection,
path: Option<PathBuf>,
migrations: &[Migration],
) -> Result<Self, StoreError> {
let journal_mode: String =
conn.query_row("PRAGMA journal_mode = WAL", [], |row| row.get(0))?;
if path.is_some() && !journal_mode.eq_ignore_ascii_case("wal") {
tracing::warn!(
path = ?path,
journal_mode = %journal_mode,
"this database did not enter WAL mode, so a reader will block \
the agent's journal writes. The usual cause is a directory \
that cannot host WAL's shared-memory file, such as a network \
mount."
);
}
conn.pragma_update(None, "synchronous", "FULL")?;
conn.busy_timeout(Duration::from_secs(5))?;
conn.pragma_update(None, "foreign_keys", true)?;
let schema_version = apply_migrations(&mut conn, migrations, &SystemClock)?;
Ok(Self {
conn: Mutex::new(conn),
path,
schema_version,
journal_mode,
clock_skew_repairs: AtomicU64::new(0),
})
}
#[must_use]
pub const fn schema_version(&self) -> u32 {
self.schema_version
}
#[must_use]
pub fn journal_mode(&self) -> &str {
&self.journal_mode
}
#[must_use]
pub fn readers_do_not_block_writers(&self) -> bool {
self.journal_mode.eq_ignore_ascii_case("wal")
}
#[must_use]
pub fn path(&self) -> Option<&Path> {
self.path.as_deref()
}
#[must_use]
pub fn clock_skew_repairs(&self) -> u64 {
self.clock_skew_repairs.load(Ordering::Relaxed)
}
pub fn dump_text(&self) -> Result<String, StoreError> {
use std::fmt::Write as _;
let conn = self.lock();
let mut out = String::new();
let _ = writeln!(out, "-- schema version {}", self.schema_version);
for table in TABLES {
let _ = writeln!(out, "-- table {table}");
let mut stmt = conn.prepare(&format!("SELECT * FROM \"{table}\""))?;
let columns: Vec<String> = stmt.column_names().into_iter().map(String::from).collect();
let mut rows = stmt.query([])?;
while let Some(row) = rows.next()? {
for (index, column) in columns.iter().enumerate() {
if index > 0 {
out.push_str(", ");
}
let _ = write!(out, "{table}.{column}={}", render(row.get_ref(index)?));
}
out.push('\n');
}
}
Ok(out)
}
fn lock(&self) -> MutexGuard<'_, Connection> {
self.conn.lock().unwrap_or_else(PoisonError::into_inner)
}
fn normalise(&self, mut fields: PersistedAttempt) -> PersistedAttempt {
if fields.last_state_change_at < fields.created_at {
tracing::warn!(
attempt = %fields.id,
created_at = %fields.created_at,
last_state_change_at = %fields.last_state_change_at,
"attempt last_state_change_at precedes created_at; the host clock \
stepped backwards. Clamping to created_at so the attempt stays \
recoverable; its recovery timeouts now measure from allocation."
);
fields.last_state_change_at = fields.created_at;
self.clock_skew_repairs.fetch_add(1, Ordering::Relaxed);
}
if let Some(terminal_at) = fields.terminal_at
&& terminal_at < fields.created_at
{
tracing::warn!(
attempt = %fields.id,
created_at = %fields.created_at,
terminal_at = %terminal_at,
"attempt terminal_at precedes created_at; the host clock stepped \
backwards. Clamping to created_at."
);
fields.terminal_at = Some(fields.created_at);
self.clock_skew_repairs.fetch_add(1, Ordering::Relaxed);
}
fields
}
fn update_policy_confirming_active_count_with(
&self,
policy: &ScalePolicy,
expected_revision: u64,
expected_active: u16,
after_write_fence: impl FnOnce(),
) -> Result<(), StoreError> {
self.update_policy_confirming_count_with(
policy,
expected_revision,
expected_active,
CountedAttempts::Active,
after_write_fence,
)
}
fn update_policy_confirming_count_with(
&self,
policy: &ScalePolicy,
expected_revision: u64,
expected_count: u16,
counted: CountedAttempts,
after_write_fence: impl FnOnce(),
) -> Result<(), StoreError> {
let fields = policy.to_persisted();
let mut params = policy_params(&fields)?;
params.push((
":expected_revision",
int(u64_to_sql("the expected revision", expected_revision)?),
));
params.push((":expected_count", int(i64::from(expected_count))));
let count_sql = counted.count_sql();
let mut conn = self.lock();
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
after_write_fence();
let changed = tx.execute(
&format!(
"UPDATE policies SET
target_scope = :target_scope,
target_slug = :target_slug,
installation_id = :installation_id,
host_id = :host_id,
requested_host_label = :requested_host_label,
routing_labels = :routing_labels,
min_capacity = :min_capacity,
max_capacity = :max_capacity,
enabled = :enabled,
state = :state,
cache_policy = :cache_policy,
workspace_mode = :workspace_mode,
workspace_path = :workspace_path,
revision = :revision
WHERE id = :id
AND revision = :expected_revision
AND :expected_count = ({count_sql})"
),
&bind(¶ms)[..],
)?;
if changed == 0 {
let revision_conflict = conflict_or_missing(&tx, fields.id, expected_revision)?;
match revision_conflict {
StoreError::StaleRevision {
expected, found, ..
} if expected == found => {
let found: i64 = tx.query_row(
&count_sql,
named_params! { ":id": uuid_text(fields.id.as_uuid()) },
|row| row.get(0),
)?;
return Err(counted.count_changed(
fields.id,
expected_count,
clamped_count(found),
));
}
other => return Err(other),
}
}
tx.commit()?;
Ok(())
}
fn attempt_from_row(&self, row: &Row<'_>) -> Result<RunnerAttempt, StoreError> {
let fields = self.normalise(persisted_attempt_from_row(row)?);
let id = fields.id;
RunnerAttempt::from_persisted(fields)
.map_err(|source| StoreError::CorruptAttempt { id, source })
}
fn attempts_where(
&self,
predicate: &str,
params: &[(&str, &dyn ToSql)],
) -> Result<Vec<RunnerAttempt>, StoreError> {
let conn = self.lock();
let mut stmt = conn.prepare(&format!(
"SELECT * FROM attempts WHERE {predicate} ORDER BY created_at, id"
))?;
let mut rows = stmt.query(params)?;
let mut out = Vec::new();
while let Some(row) = rows.next()? {
out.push(self.attempt_from_row(row)?);
}
Ok(out)
}
fn attempts_of_policy(
&self,
policy_id: PolicyId,
and: Option<&str>,
) -> Result<Vec<RunnerAttempt>, StoreError> {
let id = uuid_text(policy_id.as_uuid());
let predicate = and.map_or_else(
|| "policy_id = :policy_id".to_string(),
|and| format!("policy_id = :policy_id AND {and}"),
);
self.attempts_where(&predicate, &[(":policy_id", &id as &dyn ToSql)])
}
}
impl Store for SqliteStore {
fn put_host(&self, host: &Host) -> Result<(), StoreError> {
let conn = self.lock();
conn.execute(
"INSERT INTO hosts (
id, display_name, os, architecture, host_capacity,
service_start_mode, refresh_interval_secs, runner_root_override,
created_at
) VALUES (
:id, :display_name, :os, :architecture, :host_capacity,
:service_start_mode, :refresh_interval_secs, :runner_root_override,
:created_at
)
ON CONFLICT(id) DO UPDATE SET
display_name = excluded.display_name,
os = excluded.os,
architecture = excluded.architecture,
host_capacity = excluded.host_capacity,
service_start_mode = excluded.service_start_mode,
refresh_interval_secs = excluded.refresh_interval_secs,
runner_root_override = excluded.runner_root_override,
created_at = excluded.created_at",
named_params! {
":id": uuid_text(host.id.as_uuid()),
":display_name": host.display_name.as_str(),
":os": token(&host.os),
":architecture": token(&host.architecture),
":host_capacity": i64::from(host.host_capacity.get()),
":service_start_mode": token(&host.service_start_mode),
":refresh_interval_secs": i64::from(host.refresh_interval.as_secs()),
":runner_root_override": host
.runner_root_override
.as_ref()
.map(LocalAbsolutePath::as_str),
":created_at": timestamp_to_text(host.created_at),
},
)?;
Ok(())
}
fn host(&self, id: HostId) -> Result<Option<Host>, StoreError> {
let conn = self.lock();
let mut stmt = conn.prepare("SELECT * FROM hosts WHERE id = :id")?;
let mut rows = stmt.query(named_params! { ":id": uuid_text(id.as_uuid()) })?;
match rows.next()? {
Some(row) => Ok(Some(host_from_row(row)?)),
None => Ok(None),
}
}
fn hosts(&self) -> Result<Vec<Host>, StoreError> {
let conn = self.lock();
let mut stmt = conn.prepare("SELECT * FROM hosts ORDER BY created_at, id")?;
let mut rows = stmt.query([])?;
let mut out = Vec::new();
while let Some(row) = rows.next()? {
out.push(host_from_row(row)?);
}
Ok(out)
}
fn set_runner_root_override(
&self,
id: HostId,
expected: Option<&LocalAbsolutePath>,
new_root: Option<&LocalAbsolutePath>,
expected_uncleaned: u16,
) -> Result<(), StoreError> {
let count_sql = attempt_count_sql(&uncleaned_of_kind_sql(WorkspaceKind::Ephemeral));
let mut conn = self.lock();
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
let changed = tx.execute(
&format!(
"UPDATE hosts SET runner_root_override = :new_root
WHERE id = :id
AND runner_root_override IS :expected
AND :expected_uncleaned = ({count_sql})"
),
named_params! {
":id": uuid_text(id.as_uuid()),
":new_root": new_root.map(LocalAbsolutePath::as_str),
":expected": expected.map(LocalAbsolutePath::as_str),
":expected_uncleaned": i64::from(expected_uncleaned),
},
)?;
if changed == 0 {
let stored: Option<Option<String>> = tx
.query_row(
"SELECT runner_root_override FROM hosts WHERE id = :id",
named_params! { ":id": uuid_text(id.as_uuid()) },
|row| row.get(0),
)
.optional()?;
let Some(stored) = stored else {
return Err(StoreError::NotFound {
what: "host",
id: id.to_string(),
});
};
if stored.as_deref() != expected.map(LocalAbsolutePath::as_str) {
return Err(StoreError::RunnerRootChanged {
id,
expected: render_root(expected.map(LocalAbsolutePath::as_str)),
found: render_root(stored.as_deref()),
});
}
let found: i64 = tx.query_row(&count_sql, [], |row| row.get(0))?;
return Err(StoreError::UncleanedCountChanged {
subject: format!("host {id}"),
expected: expected_uncleaned,
found: clamped_count(found),
});
}
tx.commit()?;
Ok(())
}
fn insert_policy(&self, policy: &ScalePolicy) -> Result<(), StoreError> {
let fields = policy.to_persisted();
let params = policy_params(&fields)?;
let conn = self.lock();
conn.execute(
"INSERT INTO policies (
id, target_scope, target_slug, installation_id, host_id,
requested_host_label, routing_labels, min_capacity, max_capacity, enabled, state,
cache_policy, workspace_mode, workspace_path, revision
) VALUES (
:id, :target_scope, :target_slug, :installation_id, :host_id,
:requested_host_label, :routing_labels, :min_capacity, :max_capacity, :enabled, :state,
:cache_policy, :workspace_mode, :workspace_path, :revision
)",
&bind(¶ms)[..],
)
.map_err(|source| {
if is_constraint_violation(&source) {
StoreError::AlreadyExists {
what: "policy",
id: fields.id.to_string(),
}
} else {
StoreError::Sqlite(source)
}
})?;
Ok(())
}
fn update_policy(
&self,
policy: &ScalePolicy,
expected_revision: u64,
) -> Result<(), StoreError> {
let fields = policy.to_persisted();
let mut params = policy_params(&fields)?;
params.push((
":expected_revision",
int(u64_to_sql("the expected revision", expected_revision)?),
));
let mut conn = self.lock();
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
let changed = tx.execute(
"UPDATE policies SET
target_scope = :target_scope,
target_slug = :target_slug,
installation_id = :installation_id,
host_id = :host_id,
requested_host_label = :requested_host_label,
routing_labels = :routing_labels,
min_capacity = :min_capacity,
max_capacity = :max_capacity,
enabled = :enabled,
state = :state,
cache_policy = :cache_policy,
workspace_mode = :workspace_mode,
workspace_path = :workspace_path,
revision = :revision
WHERE id = :id AND revision = :expected_revision",
&bind(¶ms)[..],
)?;
if changed == 0 {
return Err(conflict_or_missing(&tx, fields.id, expected_revision)?);
}
tx.commit()?;
Ok(())
}
fn update_policy_confirming_active_count(
&self,
policy: &ScalePolicy,
expected_revision: u64,
expected_active: u16,
) -> Result<(), StoreError> {
self.update_policy_confirming_active_count_with(
policy,
expected_revision,
expected_active,
|| {},
)
}
fn update_policy_confirming_uncleaned_count(
&self,
policy: &ScalePolicy,
expected_revision: u64,
expected_uncleaned: u16,
) -> Result<(), StoreError> {
self.update_policy_confirming_count_with(
policy,
expected_revision,
expected_uncleaned,
CountedAttempts::Uncleaned,
|| {},
)
}
fn remove_policy(&self, id: PolicyId, expected_revision: u64) -> Result<(), StoreError> {
let mut conn = self.lock();
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
let changed = tx.execute(
"DELETE FROM policies WHERE id = :id AND revision = :expected_revision",
named_params! {
":id": uuid_text(id.as_uuid()),
":expected_revision": u64_to_sql("the expected revision", expected_revision)?,
},
)?;
if changed == 0 {
return Err(conflict_or_missing(&tx, id, expected_revision)?);
}
tx.commit()?;
Ok(())
}
fn policy(&self, id: PolicyId) -> Result<Option<ScalePolicy>, StoreError> {
let conn = self.lock();
let mut stmt = conn.prepare("SELECT * FROM policies WHERE id = :id")?;
let mut rows = stmt.query(named_params! { ":id": uuid_text(id.as_uuid()) })?;
match rows.next()? {
Some(row) => Ok(Some(policy_from_row(row)?)),
None => Ok(None),
}
}
fn policies(&self) -> Result<Vec<ScalePolicy>, StoreError> {
let conn = self.lock();
let mut stmt = conn.prepare("SELECT * FROM policies ORDER BY id")?;
let mut rows = stmt.query([])?;
let mut out = Vec::new();
while let Some(row) = rows.next()? {
out.push(policy_from_row(row)?);
}
Ok(out)
}
fn record_attempt(&self, attempt: &RunnerAttempt) -> Result<(), StoreError> {
let fields = self.normalise(attempt.to_persisted());
let params = attempt_params(&fields)?;
let conn = self.lock();
conn.execute(
"INSERT INTO attempts (
id, policy_id, github_runner_id, state, outcome, process_id,
runtime_path, workspace_mode, workspace_slot,
created_at, terminal_at, last_state_change_at
) VALUES (
:id, :policy_id, :github_runner_id, :state, :outcome, :process_id,
:runtime_path, :workspace_mode, :workspace_slot,
:created_at, :terminal_at, :last_state_change_at
)
ON CONFLICT(id) DO UPDATE SET
policy_id = excluded.policy_id,
github_runner_id = excluded.github_runner_id,
state = excluded.state,
outcome = excluded.outcome,
process_id = excluded.process_id,
runtime_path = excluded.runtime_path,
terminal_at = excluded.terminal_at,
last_state_change_at = excluded.last_state_change_at",
&bind(¶ms)[..],
)
.map_err(|source| slot_lease_error(attempt, source))?;
Ok(())
}
fn attempt(&self, id: AttemptId) -> Result<Option<RunnerAttempt>, StoreError> {
let conn = self.lock();
let mut stmt = conn.prepare("SELECT * FROM attempts WHERE id = :id")?;
let mut rows = stmt.query(named_params! { ":id": uuid_text(id.as_uuid()) })?;
match rows.next()? {
Some(row) => Ok(Some(self.attempt_from_row(row)?)),
None => Ok(None),
}
}
fn attempts(&self) -> Result<Vec<RunnerAttempt>, StoreError> {
let conn = self.lock();
let mut stmt = conn.prepare("SELECT * FROM attempts ORDER BY created_at, id")?;
let mut rows = stmt.query([])?;
let mut out = Vec::new();
while let Some(row) = rows.next()? {
out.push(self.attempt_from_row(row)?);
}
Ok(out)
}
fn attempts_for_policy(&self, policy_id: PolicyId) -> Result<Vec<RunnerAttempt>, StoreError> {
self.attempts_of_policy(policy_id, None)
}
fn active_attempts_for_policy(
&self,
policy_id: PolicyId,
) -> Result<Vec<RunnerAttempt>, StoreError> {
self.attempts_of_policy(policy_id, Some(&active_sql()))
}
fn uncleaned_attempts_for_policy(
&self,
policy_id: PolicyId,
) -> Result<Vec<RunnerAttempt>, StoreError> {
self.attempts_of_policy(policy_id, Some(&uncleaned_sql()))
}
fn slot_leases_for_policy(
&self,
policy_id: PolicyId,
) -> Result<Vec<RunnerAttempt>, StoreError> {
self.attempts_of_policy(
policy_id,
Some(&uncleaned_of_kind_sql(WorkspaceKind::Persistent)),
)
}
fn uncleaned_ephemeral_attempts(&self) -> Result<Vec<RunnerAttempt>, StoreError> {
self.attempts_where(&uncleaned_of_kind_sql(WorkspaceKind::Ephemeral), &[])
}
fn remove_attempt(&self, id: AttemptId) -> Result<bool, StoreError> {
let conn = self.lock();
let changed = conn.execute(
"DELETE FROM attempts WHERE id = :id",
named_params! { ":id": uuid_text(id.as_uuid()) },
)?;
Ok(changed > 0)
}
}
type NamedParams = Vec<(&'static str, Value)>;
fn bind(params: &NamedParams) -> Vec<(&str, &dyn ToSql)> {
params
.iter()
.map(|(name, value)| (*name, value as &dyn ToSql))
.collect()
}
fn text(value: impl Into<String>) -> Value {
Value::Text(value.into())
}
fn int(value: i64) -> Value {
Value::Integer(value)
}
fn opt_text(value: Option<String>) -> Value {
value.map_or(Value::Null, Value::Text)
}
fn opt_int(value: Option<i64>) -> Value {
value.map_or(Value::Null, Value::Integer)
}
fn policy_params(fields: &PersistedPolicy) -> Result<NamedParams, StoreError> {
Ok(vec![
(":id", text(uuid_text(fields.id.as_uuid()))),
(":target_scope", text(token(&fields.target.scope()))),
(":target_slug", text(fields.target.slug())),
(
":installation_id",
int(u64_to_sql(
"policies.installation_id",
fields.installation_id,
)?),
),
(":host_id", text(uuid_text(fields.host_id.as_uuid()))),
(
":requested_host_label",
text(fields.requested_host_label.to_string()),
),
(
":routing_labels",
opt_text(fields.routing_labels.as_ref().map(json)),
),
(":min_capacity", int(i64::from(fields.min_capacity))),
(
":max_capacity",
opt_int(fields.max_capacity.map(|m| i64::from(m.get()))),
),
(":enabled", int(i64::from(fields.enabled))),
(":state", text(token(&fields.state))),
(":cache_policy", text(token(&fields.cache_policy))),
(":workspace_mode", text(token(&fields.workspace_kind))),
(
":workspace_path",
opt_text(
fields
.workspace_root
.as_ref()
.map(|root| root.as_str().to_string()),
),
),
(
":revision",
int(u64_to_sql("policies.revision", fields.revision)?),
),
])
}
fn attempt_params(fields: &PersistedAttempt) -> Result<NamedParams, StoreError> {
let runtime_path =
fields
.runtime_path
.to_str()
.ok_or_else(|| StoreError::UnrepresentablePath {
attempt: fields.id,
path: fields.runtime_path.clone(),
})?;
Ok(vec![
(":id", text(uuid_text(fields.id.as_uuid()))),
(":policy_id", text(uuid_text(fields.policy_id.as_uuid()))),
(
":github_runner_id",
opt_int(
fields
.github_runner_id
.map(|id| u64_to_sql("attempts.github_runner_id", id))
.transpose()?,
),
),
(":state", text(token(&fields.state))),
(":outcome", opt_text(fields.outcome.as_ref().map(json))),
(":process_id", opt_int(fields.process_id.map(i64::from))),
(":runtime_path", text(runtime_path)),
(":workspace_mode", text(token(&fields.workspace_kind))),
(
":workspace_slot",
opt_int(fields.workspace_slot.map(i64::from)),
),
(":created_at", text(timestamp_to_text(fields.created_at))),
(
":terminal_at",
opt_text(fields.terminal_at.map(timestamp_to_text)),
),
(
":last_state_change_at",
text(timestamp_to_text(fields.last_state_change_at)),
),
])
}
fn host_from_row(row: &Row<'_>) -> Result<Host, StoreError> {
const TABLE: &str = "hosts";
let id = HostId::from_uuid(uuid_column(row, TABLE, "id")?);
let key = id.to_string();
let display_name: String = row.get("display_name")?;
let os: Os = token_column(row, TABLE, "os", &key)?;
let architecture: Arch = token_column(row, TABLE, "architecture", &key)?;
let host_capacity = NonZeroU16::new(u16_column(row, TABLE, "host_capacity", &key)?).ok_or(
StoreError::CorruptColumn {
table: TABLE,
column: "host_capacity",
id: key.clone(),
value: "0".to_string(),
expected: "a non-zero capacity; a host that declares zero is not a \
configured host but a disabled one",
},
)?;
let service_start_mode: StartMode = token_column(row, TABLE, "service_start_mode", &key)?;
let refresh_interval_secs = u16_column(row, TABLE, "refresh_interval_secs", &key)?;
let runner_root_override = row
.get::<_, Option<String>>("runner_root_override")?
.map(LocalAbsolutePath::new)
.transpose()
.map_err(|source| StoreError::CorruptHostWorkspace {
id,
source: WorkspaceError::from(source),
})?;
let created_at = timestamp_column(row, TABLE, "created_at", &key)?;
let mut host = Host::new(
id,
&display_name,
os,
architecture,
host_capacity,
created_at,
)
.map_err(|source| StoreError::CorruptHost { id, source })?;
host.service_start_mode = service_start_mode;
host.refresh_interval = RefreshInterval::from_secs(refresh_interval_secs)
.map_err(|source| StoreError::CorruptHost { id, source })?;
host.runner_root_override = runner_root_override;
Ok(host)
}
fn policy_from_row(row: &Row<'_>) -> Result<ScalePolicy, StoreError> {
const TABLE: &str = "policies";
let id = PolicyId::from_uuid(uuid_column(row, TABLE, "id")?);
let key = id.to_string();
let scope: TargetScope = token_column(row, TABLE, "target_scope", &key)?;
let slug: String = row.get("target_slug")?;
let target = match scope {
TargetScope::Repository => ScaleTarget::repository(&slug),
TargetScope::Organization => ScaleTarget::organization(&slug),
}
.map_err(|source| StoreError::CorruptPolicy {
id,
source: PolicyError::Invalid(source),
})?;
let routing_labels: Option<RoutingLabels> =
json_column(row, TABLE, "routing_labels", &key, "a routing label set")?;
let max_capacity = match u16_option_column(row, TABLE, "max_capacity", &key)? {
Some(raw) => Some(NonZeroU16::new(raw).ok_or(StoreError::CorruptColumn {
table: TABLE,
column: "max_capacity",
id: key.clone(),
value: "0".to_string(),
expected: "a non-zero ceiling; an autoscale policy that may start no \
runner is a monitor-only policy and stores NULL here",
})?),
None => None,
};
let fields = PersistedPolicy {
id,
target,
installation_id: u64_column(row, TABLE, "installation_id", &key)?,
host_id: HostId::from_uuid(uuid_column(row, TABLE, "host_id")?),
requested_host_label: HostLabel::new(row.get::<_, String>("requested_host_label")?)
.map_err(|source| StoreError::CorruptPolicy {
id,
source: PolicyError::Invalid(source),
})?,
routing_labels,
min_capacity: u16_column(row, TABLE, "min_capacity", &key)?,
max_capacity,
enabled: bool_column(row, TABLE, "enabled", &key)?,
state: token_column::<PolicyState>(row, TABLE, "state", &key)?,
cache_policy: token_column::<CachePolicy>(row, TABLE, "cache_policy", &key)?,
workspace_kind: token_column::<WorkspaceKind>(row, TABLE, "workspace_mode", &key)?,
workspace_root: row
.get::<_, Option<String>>("workspace_path")?
.map(LocalAbsolutePath::new)
.transpose()
.map_err(|source| StoreError::CorruptPolicy {
id,
source: PolicyError::Workspace(WorkspaceError::from(source)),
})?,
revision: u64_column(row, TABLE, "revision", &key)?,
};
ScalePolicy::from_persisted(fields).map_err(|source| StoreError::CorruptPolicy { id, source })
}
fn persisted_attempt_from_row(row: &Row<'_>) -> Result<PersistedAttempt, StoreError> {
const TABLE: &str = "attempts";
let id = AttemptId::from_uuid(uuid_column(row, TABLE, "id")?);
let key = id.to_string();
let runtime_path: String = row.get("runtime_path")?;
Ok(PersistedAttempt {
id,
policy_id: PolicyId::from_uuid(uuid_column(row, TABLE, "policy_id")?),
github_runner_id: u64_option_column(row, TABLE, "github_runner_id", &key)?,
state: token_column::<AttemptState>(row, TABLE, "state", &key)?,
outcome: json_column::<AttemptOutcome>(row, TABLE, "outcome", &key, "an attempt outcome")?,
process_id: u32_option_column(row, TABLE, "process_id", &key)?,
runtime_path: PathBuf::from(runtime_path),
workspace_kind: token_column::<WorkspaceKind>(row, TABLE, "workspace_mode", &key)?,
workspace_slot: u16_option_column(row, TABLE, "workspace_slot", &key)?,
created_at: timestamp_column(row, TABLE, "created_at", &key)?,
terminal_at: timestamp_option_column(row, TABLE, "terminal_at", &key)?,
last_state_change_at: timestamp_column(row, TABLE, "last_state_change_at", &key)?,
})
}
fn conflict_or_missing(
tx: &rusqlite::Transaction<'_>,
id: PolicyId,
expected: u64,
) -> Result<StoreError, StoreError> {
let found: Option<i64> = tx
.query_row(
"SELECT revision FROM policies WHERE id = :id",
named_params! { ":id": uuid_text(id.as_uuid()) },
|row| row.get(0),
)
.optional()?;
Ok(match found {
Some(found) => match u64::try_from(found) {
Ok(found) => StoreError::StaleRevision {
id,
expected,
found,
},
Err(_) => StoreError::CorruptColumn {
table: "policies",
column: "revision",
id: id.to_string(),
value: clip(&found.to_string()),
expected: "a non-negative integer",
},
},
None => StoreError::NotFound {
what: "policy",
id: id.to_string(),
},
})
}
fn token<T: Serialize + fmt::Debug>(value: &T) -> String {
match serde_json::to_value(value) {
Ok(serde_json::Value::String(token)) => token,
other => panic!(
"{value:?} must serialise to a JSON string to be stored as a column \
token, got {other:?}"
),
}
}
fn json<T: Serialize + fmt::Debug>(value: &T) -> String {
serde_json::to_string(value)
.unwrap_or_else(|e| panic!("{value:?} must serialise to JSON for storage: {e}"))
}
fn timestamp_to_text(value: Timestamp) -> String {
value.to_rfc3339_opts(chrono::SecondsFormat::Nanos, true)
}
fn uuid_text(value: &Uuid) -> String {
value.hyphenated().to_string()
}
fn u64_to_sql(what: &'static str, value: u64) -> Result<i64, StoreError> {
i64::try_from(value).map_err(|_| StoreError::UnrepresentableInteger { what, value })
}
pub const ECHO_LIMIT: usize = 60;
const FREE_FORM_COLUMNS: &[(&str, &str)] = &[("attempts", "outcome")];
fn carries_free_form_text(table: &str, column: &str) -> bool {
FREE_FORM_COLUMNS
.iter()
.any(|(t, c)| *t == table && *c == column)
}
fn clip(raw: &str) -> String {
match raw.char_indices().nth(ECHO_LIMIT) {
None => raw.to_string(),
Some((cut, _)) => format!(
"{}... ({} bytes in total, truncated)",
&raw[..cut],
raw.len()
),
}
}
fn position_only(raw: &str, error: &serde_json::Error) -> String {
if error.line() == 0 {
format!(
"a {}-byte payload that is not echoed (serde records no position \
for this failure, so where in the payload it went wrong is not \
known)",
raw.len()
)
} else {
format!(
"a {}-byte payload that is not echoed (it stops parsing at line {}, column {})",
raw.len(),
error.line(),
error.column()
)
}
}
fn render_root(value: Option<&str>) -> String {
value.map_or_else(|| "the platform default".to_string(), clip)
}
fn slot_lease_error(attempt: &RunnerAttempt, source: rusqlite::Error) -> StoreError {
match (
attempt.workspace().slot_number(),
is_constraint_violation(&source),
) {
(Some(slot), true) if attempt.holds_slot_lease() => StoreError::SlotAlreadyLeased {
policy: attempt.policy_id,
slot,
},
_ => StoreError::Sqlite(source),
}
}
fn is_constraint_violation(error: &rusqlite::Error) -> bool {
matches!(
error,
rusqlite::Error::SqliteFailure(inner, _)
if inner.code == rusqlite::ErrorCode::ConstraintViolation
)
}
fn render(value: ValueRef<'_>) -> String {
match value {
ValueRef::Null => "NULL".to_string(),
ValueRef::Integer(i) => i.to_string(),
ValueRef::Real(f) => f.to_string(),
ValueRef::Text(bytes) => String::from_utf8_lossy(bytes).into_owned(),
ValueRef::Blob(bytes) => bytes.iter().map(|b| format!("{b:02x}")).collect(),
}
}
fn uuid_column(
row: &Row<'_>,
table: &'static str,
column: &'static str,
) -> Result<Uuid, StoreError> {
let raw: String = row.get(column)?;
Uuid::parse_str(&raw).map_err(|_| StoreError::CorruptColumn {
table,
column,
id: clip(&raw),
value: clip(&raw),
expected: "a hyphenated UUID",
})
}
fn token_column<T: DeserializeOwned>(
row: &Row<'_>,
table: &'static str,
column: &'static str,
id: &str,
) -> Result<T, StoreError> {
let raw: String = row.get(column)?;
serde_json::from_value(serde_json::Value::String(raw.clone())).map_err(|_| {
StoreError::CorruptColumn {
table,
column,
id: id.to_string(),
value: clip(&raw),
expected: "one of this column's recognised tokens",
}
})
}
fn json_column<T: DeserializeOwned>(
row: &Row<'_>,
table: &'static str,
column: &'static str,
id: &str,
expected: &'static str,
) -> Result<Option<T>, StoreError> {
match row.get::<_, Option<String>>(column)? {
None => Ok(None),
Some(raw) => {
serde_json::from_str(&raw)
.map(Some)
.map_err(|error| StoreError::CorruptColumn {
table,
column,
id: id.to_string(),
value: if carries_free_form_text(table, column) {
position_only(&raw, &error)
} else {
clip(&raw)
},
expected,
})
}
}
}
fn timestamp_column(
row: &Row<'_>,
table: &'static str,
column: &'static str,
id: &str,
) -> Result<Timestamp, StoreError> {
let raw: String = row.get(column)?;
parse_timestamp(&raw, table, column, id)
}
fn timestamp_option_column(
row: &Row<'_>,
table: &'static str,
column: &'static str,
id: &str,
) -> Result<Option<Timestamp>, StoreError> {
match row.get::<_, Option<String>>(column)? {
None => Ok(None),
Some(raw) => parse_timestamp(&raw, table, column, id).map(Some),
}
}
fn parse_timestamp(
raw: &str,
table: &'static str,
column: &'static str,
id: &str,
) -> Result<Timestamp, StoreError> {
chrono::DateTime::parse_from_rfc3339(raw)
.map(|value| value.with_timezone(&chrono::Utc))
.map_err(|_| StoreError::CorruptColumn {
table,
column,
id: id.to_string(),
value: clip(raw),
expected: "an RFC 3339 timestamp",
})
}
fn bool_column(
row: &Row<'_>,
table: &'static str,
column: &'static str,
id: &str,
) -> Result<bool, StoreError> {
match row.get::<_, i64>(column)? {
0 => Ok(false),
1 => Ok(true),
other => Err(StoreError::CorruptColumn {
table,
column,
id: id.to_string(),
value: other.to_string(),
expected: "0 or 1",
}),
}
}
macro_rules! integer_column {
($name:ident, $ty:ty, $expected:literal) => {
fn $name(
row: &Row<'_>,
table: &'static str,
column: &'static str,
id: &str,
) -> Result<$ty, StoreError> {
let raw: i64 = row.get(column)?;
<$ty>::try_from(raw).map_err(|_| StoreError::CorruptColumn {
table,
column,
id: id.to_string(),
value: raw.to_string(),
expected: $expected,
})
}
};
}
macro_rules! integer_option_column {
($name:ident, $ty:ty, $expected:literal) => {
fn $name(
row: &Row<'_>,
table: &'static str,
column: &'static str,
id: &str,
) -> Result<Option<$ty>, StoreError> {
match row.get::<_, Option<i64>>(column)? {
None => Ok(None),
Some(raw) => {
<$ty>::try_from(raw)
.map(Some)
.map_err(|_| StoreError::CorruptColumn {
table,
column,
id: id.to_string(),
value: raw.to_string(),
expected: $expected,
})
}
}
}
};
}
integer_column!(u16_column, u16, "a value in 0..=65535");
integer_column!(u64_column, u64, "a non-negative integer");
integer_option_column!(u16_option_column, u16, "a value in 0..=65535");
integer_option_column!(u32_option_column, u32, "a value in 0..=4294967295");
integer_option_column!(u64_option_column, u64, "a non-negative integer");
#[cfg(test)]
mod tests {
use super::*;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, mpsc};
use std::time::{Duration, Instant};
use crate::attempt::FailureReason;
use crate::model::Label;
use crate::policy::PolicyMode;
use crate::workspace::{AttemptWorkspace, WorkspacePolicy};
const HOST_UUID: &str = "00000000-0000-0000-0000-000000000001";
const POLICY_UUID: &str = "00000000-0000-0000-0000-000000000010";
const ATTEMPT_UUID: &str = "00000000-0000-0000-0000-000000000100";
const HISTORICAL_PATH: &str = "runtime/policy/pre-upgrade-attempt";
const LABELS_JSON: &str = r#"{"host_label":"rm-home-win-x64","additional":[]}"#;
const COMPLETED_JOB: &str = r#"{"outcome":"completed_job"}"#;
static ATTEMPT_WRITE_BLOCKED: AtomicBool = AtomicBool::new(false);
fn mark_attempt_write_blocked(_: i32) -> bool {
ATTEMPT_WRITE_BLOCKED.store(true, Ordering::Release);
true
}
fn host_id() -> HostId {
HostId::from_u128(0x0000_0001)
}
fn policy_id() -> PolicyId {
PolicyId::from_u128(0x0000_0010)
}
fn attempt_id() -> AttemptId {
AttemptId::from_u128(0x0000_0100)
}
fn ts(secs: i64) -> Timestamp {
chrono::DateTime::from_timestamp(secs, 0).expect("a representable instant")
}
fn store() -> SqliteStore {
SqliteStore::open_in_memory().expect("an in-memory database always opens")
}
#[derive(Debug, Clone)]
struct RawHost {
id: String,
display_name: String,
os: String,
architecture: String,
host_capacity: i64,
service_start_mode: String,
refresh_interval_secs: i64,
runner_root_override: Option<String>,
created_at: String,
}
impl Default for RawHost {
fn default() -> Self {
Self {
id: HOST_UUID.to_string(),
display_name: "home-pc".to_string(),
os: "windows".to_string(),
architecture: "x64".to_string(),
host_capacity: 2,
service_start_mode: "boot".to_string(),
refresh_interval_secs: 60,
runner_root_override: None,
created_at: timestamp_to_text(ts(1_000)),
}
}
}
impl RawHost {
fn insert(&self, store: &SqliteStore) {
store
.lock()
.execute(
"INSERT OR REPLACE INTO hosts (
id, display_name, os, architecture, host_capacity,
service_start_mode, refresh_interval_secs,
runner_root_override, created_at
) VALUES (
:id, :display_name, :os, :architecture, :host_capacity,
:service_start_mode, :refresh_interval_secs,
:runner_root_override, :created_at
)",
named_params! {
":id": self.id,
":display_name": self.display_name,
":os": self.os,
":architecture": self.architecture,
":host_capacity": self.host_capacity,
":service_start_mode": self.service_start_mode,
":refresh_interval_secs": self.refresh_interval_secs,
":runner_root_override": self.runner_root_override,
":created_at": self.created_at,
},
)
.expect("the raw host row is writable");
}
}
#[derive(Debug, Clone)]
struct RawPolicy {
id: String,
target_scope: String,
target_slug: String,
installation_id: i64,
host_id: String,
routing_labels: Option<String>,
min_capacity: i64,
max_capacity: Option<i64>,
enabled: i64,
state: String,
cache_policy: String,
workspace_mode: String,
workspace_path: Option<String>,
revision: i64,
}
impl Default for RawPolicy {
fn default() -> Self {
Self {
id: POLICY_UUID.to_string(),
target_scope: "repository".to_string(),
target_slug: "o/r".to_string(),
installation_id: 1,
host_id: HOST_UUID.to_string(),
routing_labels: Some(LABELS_JSON.to_string()),
min_capacity: 0,
max_capacity: Some(2),
enabled: 1,
state: "active".to_string(),
cache_policy: "retain_runner_package".to_string(),
workspace_mode: "ephemeral".to_string(),
workspace_path: None,
revision: 1,
}
}
}
impl RawPolicy {
fn insert(&self, store: &SqliteStore) {
store
.lock()
.execute(
"INSERT OR REPLACE INTO policies (
id, target_scope, target_slug, installation_id, host_id,
routing_labels, min_capacity, max_capacity, enabled,
state, cache_policy, workspace_mode, workspace_path, revision
) VALUES (
:id, :target_scope, :target_slug, :installation_id, :host_id,
:routing_labels, :min_capacity, :max_capacity, :enabled,
:state, :cache_policy, :workspace_mode, :workspace_path, :revision
)",
named_params! {
":id": self.id,
":target_scope": self.target_scope,
":target_slug": self.target_slug,
":installation_id": self.installation_id,
":host_id": self.host_id,
":routing_labels": self.routing_labels,
":min_capacity": self.min_capacity,
":max_capacity": self.max_capacity,
":enabled": self.enabled,
":state": self.state,
":cache_policy": self.cache_policy,
":workspace_mode": self.workspace_mode,
":workspace_path": self.workspace_path,
":revision": self.revision,
},
)
.expect("the raw policy row is writable");
}
}
#[derive(Debug, Clone)]
struct RawAttempt {
id: String,
policy_id: String,
github_runner_id: Option<i64>,
state: String,
outcome: Option<String>,
process_id: Option<i64>,
runtime_path: String,
workspace_mode: String,
workspace_slot: Option<i64>,
created_at: String,
terminal_at: Option<String>,
last_state_change_at: String,
}
impl Default for RawAttempt {
fn default() -> Self {
Self {
id: ATTEMPT_UUID.to_string(),
policy_id: POLICY_UUID.to_string(),
github_runner_id: None,
state: "allocated".to_string(),
outcome: None,
process_id: None,
runtime_path: "runtime/policy/attempt".to_string(),
workspace_mode: "ephemeral".to_string(),
workspace_slot: None,
created_at: timestamp_to_text(ts(1_000)),
terminal_at: None,
last_state_change_at: timestamp_to_text(ts(1_000)),
}
}
}
impl RawAttempt {
fn insert(&self, store: &SqliteStore) {
store
.lock()
.execute(
"INSERT OR REPLACE INTO attempts (
id, policy_id, github_runner_id, state, outcome, process_id,
runtime_path, workspace_mode, workspace_slot,
created_at, terminal_at, last_state_change_at
) VALUES (
:id, :policy_id, :github_runner_id, :state, :outcome, :process_id,
:runtime_path, :workspace_mode, :workspace_slot,
:created_at, :terminal_at, :last_state_change_at
)",
named_params! {
":id": self.id,
":policy_id": self.policy_id,
":github_runner_id": self.github_runner_id,
":state": self.state,
":outcome": self.outcome,
":process_id": self.process_id,
":runtime_path": self.runtime_path,
":workspace_mode": self.workspace_mode,
":workspace_slot": self.workspace_slot,
":created_at": self.created_at,
":terminal_at": self.terminal_at,
":last_state_change_at": self.last_state_change_at,
},
)
.expect("the raw attempt row is writable");
}
}
fn a_root(leaf: &str) -> LocalAbsolutePath {
let raw = if cfg!(windows) {
format!("X:\\{leaf}")
} else {
format!("/{leaf}")
};
LocalAbsolutePath::new(raw).expect("a fixture root is a storable local path")
}
#[test]
fn the_on_disk_tokens_are_pinned() {
for (state, expected) in [
(AttemptState::Allocated, "allocated"),
(AttemptState::JitReceived, "jit_received"),
(AttemptState::Starting, "starting"),
(AttemptState::Idle, "idle"),
(AttemptState::Busy, "busy"),
(AttemptState::Finished, "finished"),
(AttemptState::Failed, "failed"),
(AttemptState::Orphaned, "orphaned"),
(AttemptState::Cleaned, "cleaned"),
] {
assert_eq!(token(&state), expected);
}
assert_eq!(
AttemptState::ALL.len(),
9,
"a new AttemptState needs a pinned token above"
);
for (state, expected) in [
(PolicyState::Pending, "pending"),
(PolicyState::Active, "active"),
(PolicyState::Draining, "draining"),
(PolicyState::Disabled, "disabled"),
(PolicyState::RepairRequired, "repair_required"),
(PolicyState::AuthenticationFailed, "authentication_failed"),
] {
assert_eq!(token(&state), expected);
}
assert_eq!(
PolicyState::ALL.len(),
6,
"a new PolicyState needs a pinned token above"
);
for os in Os::ALL {
assert_eq!(
token(&os),
match os {
Os::Windows => "windows",
Os::MacOs => "mac_os",
Os::Linux => "linux",
}
);
}
for arch in Arch::ALL {
assert_eq!(
token(&arch),
match arch {
Arch::X64 => "x64",
Arch::Arm64 => "arm64",
Arch::Arm32 => "arm32",
}
);
}
for mode in [StartMode::Boot, StartMode::Login] {
assert_eq!(
token(&mode),
match mode {
StartMode::Boot => "boot",
StartMode::Login => "login",
}
);
}
for cache in [
CachePolicy::RetainRunnerPackage,
CachePolicy::DiscardRunnerPackage,
] {
assert_eq!(
token(&cache),
match cache {
CachePolicy::RetainRunnerPackage => "retain_runner_package",
CachePolicy::DiscardRunnerPackage => "discard_runner_package",
}
);
}
for scope in [TargetScope::Repository, TargetScope::Organization] {
assert_eq!(
token(&scope),
match scope {
TargetScope::Repository => "repository",
TargetScope::Organization => "organization",
}
);
}
for kind in [WorkspaceKind::Ephemeral, WorkspaceKind::Persistent] {
assert_eq!(
token(&kind),
match kind {
WorkspaceKind::Ephemeral => "ephemeral",
WorkspaceKind::Persistent => "persistent",
}
);
}
assert_eq!(Os::Windows.to_string(), "win");
assert_eq!(token(&Os::Windows), "windows");
}
#[test]
fn a_timestamp_round_trips_to_the_nanosecond() {
let precise = chrono::DateTime::from_timestamp(1_787_270_400, 123_456_789)
.expect("a representable instant");
let text = timestamp_to_text(precise);
assert_eq!(text, "2026-08-21T00:00:00.123456789Z");
assert_eq!(
parse_timestamp(&text, "t", "c", "id").expect("round trips"),
precise
);
assert_eq!(timestamp_to_text(ts(0)).len(), text.len());
assert!(timestamp_to_text(ts(0)) < timestamp_to_text(ts(1)));
}
#[test]
fn the_migration_chain_is_ordered_and_starts_at_one() {
assert!(!MIGRATIONS.is_empty());
assert_eq!(MIGRATIONS[0].version, 1);
for pair in MIGRATIONS.windows(2) {
assert!(
pair[1].version > pair[0].version,
"the chain must be strictly ascending; {} does not follow {}",
pair[1].version,
pair[0].version
);
}
assert_eq!(
MIGRATIONS.last().expect("non-empty").version,
SCHEMA_VERSION,
"SCHEMA_VERSION must be the last step in the chain, or a fresh \
database reports a version it was never migrated to"
);
}
#[test]
fn a_fresh_database_gets_the_whole_chain() {
let store = store();
assert_eq!(store.schema_version(), SCHEMA_VERSION);
let conn = store.lock();
for table in TABLES {
let count: i64 = conn
.query_row(
"SELECT count(*) FROM sqlite_master WHERE type = 'table' AND name = ?1",
[table],
|row| row.get(0),
)
.expect("sqlite_master is readable");
assert_eq!(count, 1, "{table} was not created");
}
let mut stmt = conn
.prepare("SELECT version FROM schema_migrations ORDER BY version")
.expect("prepared");
let applied: Vec<i64> = stmt
.query_map([], |row| row.get(0))
.expect("queried")
.collect::<Result<_, _>>()
.expect("collected");
assert_eq!(
applied,
MIGRATIONS
.iter()
.map(|m| i64::from(m.version))
.collect::<Vec<_>>()
);
}
#[test]
fn a_database_one_version_behind_gets_only_the_missing_step() {
const CHAIN: &[Migration] = &[
Migration {
version: 1,
name: "first",
sql: "CREATE TABLE step_one (id INTEGER NOT NULL PRIMARY KEY, \
note TEXT NOT NULL) STRICT;",
},
Migration {
version: 2,
name: "second",
sql: "CREATE TABLE step_two (id INTEGER NOT NULL PRIMARY KEY) STRICT;",
},
];
let mut conn = Connection::open_in_memory().expect("in-memory");
assert_eq!(
apply_migrations(&mut conn, &CHAIN[..1], &SystemClock).expect("step one applies"),
1
);
conn.execute(
"INSERT INTO step_one (id, note) VALUES (1, 'written between the two steps')",
[],
)
.expect("insertable");
assert_eq!(
apply_migrations(&mut conn, CHAIN, &SystemClock).expect("only step two applies"),
2
);
let note: String = conn
.query_row("SELECT note FROM step_one WHERE id = 1", [], |row| {
row.get(0)
})
.expect("the row written before the second step survives it");
assert_eq!(note, "written between the two steps");
let two: i64 = conn
.query_row("SELECT count(*) FROM step_two", [], |row| row.get(0))
.expect("step two created its table");
assert_eq!(two, 0);
assert_eq!(current_version(&conn).expect("readable"), 2);
assert_eq!(
apply_migrations(&mut conn, CHAIN, &SystemClock).expect("idempotent"),
2
);
let applied: i64 = conn
.query_row("SELECT count(*) FROM schema_migrations", [], |row| {
row.get(0)
})
.expect("readable");
assert_eq!(applied, 2, "a step must be recorded exactly once");
}
fn a_database_at_version(path: &Path, version: u32) {
let mut conn = Connection::open(path).expect("openable");
let applied = apply_migrations(&mut conn, &MIGRATIONS[..version as usize], &SystemClock)
.expect("the older chain applies");
assert_eq!(applied, version);
conn.execute(
"INSERT INTO hosts (
id, display_name, os, architecture, host_capacity,
service_start_mode, refresh_interval_secs, created_at
) VALUES (?1, 'home-pc', 'windows', 'x64', 2, 'boot', 60, ?2)",
rusqlite::params![HOST_UUID, timestamp_to_text(ts(1_000))],
)
.expect("a version-1 host row");
let policy_sql = if version >= 2 {
"INSERT INTO policies (
id, target_scope, target_slug, installation_id, host_id,
requested_host_label, routing_labels, min_capacity, max_capacity,
enabled, state, cache_policy, revision
) VALUES (?1, 'repository', 'o/r', 1, ?2, 'host', ?3, 0, 2, 1,
'active', 'retain_runner_package', 1)"
} else {
"INSERT INTO policies (
id, target_scope, target_slug, installation_id, host_id,
routing_labels, min_capacity, max_capacity,
enabled, state, cache_policy, revision
) VALUES (?1, 'repository', 'o/r', 1, ?2, ?3, 0, 2, 1,
'active', 'retain_runner_package', 1)"
};
conn.execute(
policy_sql,
rusqlite::params![POLICY_UUID, HOST_UUID, LABELS_JSON],
)
.expect("a historical policy row");
conn.execute(
"INSERT INTO attempts (
id, policy_id, state, runtime_path, created_at, last_state_change_at
) VALUES (?1, ?2, 'idle', ?3, ?4, ?4)",
rusqlite::params![
ATTEMPT_UUID,
POLICY_UUID,
HISTORICAL_PATH,
timestamp_to_text(ts(1_000))
],
)
.expect("a historical attempt row");
drop(conn);
}
fn assert_everything_migrated_to_ephemeral(store: &SqliteStore) {
assert_eq!(store.schema_version(), SCHEMA_VERSION);
let host = store.host(host_id()).expect("loads").expect("present");
assert_eq!(
host.runner_root_override, None,
"a migrated host is on the platform default; storing the effective \
path instead would freeze today's default into every database"
);
assert!(!host.has_configured_runner_root());
assert_eq!(host.host_capacity.get(), 2);
assert_eq!(host.service_start_mode, StartMode::Boot);
let policy = store.policy(policy_id()).expect("loads").expect("present");
assert_eq!(
policy.workspace_policy(),
&WorkspacePolicy::Ephemeral,
"an upgrade must not retain a workspace the operator never selected"
);
assert_eq!(policy.requested_host_label.as_str(), "host");
let attempt = store
.attempt(attempt_id())
.expect("loads")
.expect("present");
assert_eq!(attempt.workspace(), AttemptWorkspace::Ephemeral);
assert!(!attempt.holds_slot_lease());
assert_eq!(
attempt.runtime_path(),
Path::new(HISTORICAL_PATH),
"`No journal row is rewritten merely to adopt the new default`: \
recovery removes the exact directory the attempt was created in, \
so a rewritten path would point cleanup at one it never used"
);
assert_eq!(attempt.state(), AttemptState::Idle);
}
#[test]
fn a_version_one_database_migrates_through_the_whole_chain() {
let dir = tempfile::tempdir().expect("a temporary directory");
let path = dir.path().join("runner-manager.sqlite3");
a_database_at_version(&path, 1);
let store = SqliteStore::open(&path).expect("a version-1 database migrates");
assert_everything_migrated_to_ephemeral(&store);
let conn = store.lock();
let mut stmt = conn
.prepare("SELECT version FROM schema_migrations ORDER BY version")
.expect("prepared");
let applied: Vec<i64> = stmt
.query_map([], |row| row.get(0))
.expect("queried")
.collect::<Result<_, _>>()
.expect("collected");
assert_eq!(applied, vec![1, 2, 3], "the full chain, in order, once");
}
#[test]
fn a_version_two_database_migrates_every_row_to_ephemeral() {
let dir = tempfile::tempdir().expect("a temporary directory");
let path = dir.path().join("runner-manager.sqlite3");
a_database_at_version(&path, 2);
let store = SqliteStore::open(&path).expect("a version-2 database migrates");
assert_everything_migrated_to_ephemeral(&store);
drop(store);
let reopened = SqliteStore::open(&path).expect("reopens");
assert_everything_migrated_to_ephemeral(&reopened);
}
#[test]
fn a_database_from_a_newer_build_is_refused_rather_than_guessed_at() {
let dir = tempfile::tempdir().expect("a temporary directory");
let path = dir.path().join("runner-manager.sqlite3");
let store = SqliteStore::open(&path).expect("a fresh database opens");
assert_eq!(store.schema_version(), SCHEMA_VERSION);
drop(store);
let future = SCHEMA_VERSION + 1;
{
let conn = Connection::open(&path).expect("reopenable");
conn.execute(
"INSERT INTO schema_migrations (version, name, applied_at) \
VALUES (?1, 'from_the_future', ?2)",
rusqlite::params![i64::from(future), timestamp_to_text(ts(2_000))],
)
.expect("insertable");
}
let error = SqliteStore::open(&path).expect_err("a newer database must be refused");
assert!(
matches!(
error,
StoreError::SchemaTooNew { found, supported }
if found == future && supported == SCHEMA_VERSION
),
"expected SchemaTooNew, got {error:?}"
);
let message = error.to_string();
assert!(
message.contains(&future.to_string()) && message.contains(&SCHEMA_VERSION.to_string()),
"the error must name both versions so an operator knows which way to \
move: {message}"
);
assert!(
!error.is_conflict(),
"a schema refusal is not an optimistic-concurrency conflict"
);
}
#[test]
fn a_corrupt_schema_version_is_named_rather_than_reported_as_four_billion() {
let mut conn = Connection::open_in_memory().expect("in-memory");
conn.execute_batch(BOOTSTRAP_SQL).expect("bootstrapped");
conn.execute(
"INSERT INTO schema_migrations (version, name, applied_at) \
VALUES (-1, 'hand_edited', ?1)",
rusqlite::params![timestamp_to_text(ts(2_000))],
)
.expect("insertable");
let error = current_version(&conn).expect_err("a negative version is not a version");
assert!(
matches!(
&error,
StoreError::CorruptColumn {
table: "schema_migrations",
column: "version",
..
}
),
"expected a named corrupt column, got {error:?}"
);
let message = error.to_string();
assert!(
message.contains("-1") && !message.contains(&u32::MAX.to_string()),
"the message must name the row's actual value: {message}"
);
assert!(apply_migrations(&mut conn, MIGRATIONS, &SystemClock).is_err());
}
#[test]
fn a_negative_stored_revision_is_a_corrupt_column_and_not_revision_zero() {
let store = store();
RawPolicy {
revision: -1,
..RawPolicy::default()
}
.insert(&store);
let policy = ScalePolicy::new(
policy_id(),
ScaleTarget::repository("o/r").expect("valid"),
1,
host_id(),
PolicyMode::autoscale(
RoutingLabels::from_host_label(Label::new("rm-home-win-x64").expect("valid")),
0,
NonZeroU16::new(2).expect("non-zero"),
)
.expect("valid"),
CachePolicy::default(),
);
let error = store
.update_policy(&policy, 0)
.expect_err("the row's revision is not 0, so this matches nothing");
assert!(
matches!(
&error,
StoreError::CorruptColumn {
table: "policies",
column: "revision",
..
}
),
"expected a named corrupt column, got {error:?}"
);
assert!(
!error.is_conflict(),
"a corrupt row is not a lost race, and a caller told to re-read and \
retry would loop for ever on it"
);
assert!(
error.to_string().contains("-1"),
"the message must name the value that needs fixing: {error}"
);
}
#[test]
fn a_corrupt_column_error_clips_the_payload_it_echoes() {
assert_eq!(clip("short"), "short");
let exact = "a".repeat(ECHO_LIMIT);
assert_eq!(clip(&exact), exact, "the limit is an edge, not a target");
let over = "a".repeat(ECHO_LIMIT + 1);
let clipped = clip(&over);
assert!(clipped.starts_with(&exact));
assert!(
clipped.contains(&format!("{} bytes in total", over.len())),
"a clipped echo must say it is one: {clipped}"
);
let wide = "é".repeat(ECHO_LIMIT * 2);
assert!(clip(&wide).starts_with(&"é".repeat(ECHO_LIMIT)));
}
#[test]
fn a_secret_in_a_free_form_column_is_not_echoed_whole_into_the_error() {
let store = store();
let blob = "QUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVphYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3h5ejAxMjM0\
NTY3ODkrLwABAgMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fICEiIyQlJicoKSorLC0uLzA"
.to_string();
assert!(blob.len() > ECHO_LIMIT * 2);
RawAttempt {
state: "finished".to_string(),
outcome: Some(format!(r#"{{"outcome":"went_home","detail":"{blob}"}}"#)),
terminal_at: Some(timestamp_to_text(ts(2_000))),
..RawAttempt::default()
}
.insert(&store);
let error = store
.attempt(attempt_id())
.expect_err("`went_home` is not an outcome");
let StoreError::CorruptColumn { value, .. } = &error else {
panic!("expected a corrupt column, got {error:?}");
};
assert!(
!value.contains(&blob),
"the whole payload must not be repeated: {value}"
);
assert!(
value.chars().count() < blob.chars().count(),
"the echo must be shorter than what it echoes"
);
assert!(
!value.contains(&blob[..8]),
"no leading fragment of the payload may travel either: {value}"
);
assert!(
error.to_string().contains(&attempt_id().to_string()),
"the error must name the row to fix: {error}"
);
}
#[test]
fn a_short_secret_in_the_free_form_column_is_not_echoed_at_all() {
let planted = "ghs_9tokenish";
let raw = format!(r#"{{"outcome":"failed","reason":{{"other":"{planted}"}}"#);
assert!(raw.chars().count() < ECHO_LIMIT);
assert_eq!(
clip(&raw),
raw,
"a sixty-character budget repeats this row in full, which is what \
makes the budget the wrong instrument at this column"
);
assert!(carries_free_form_text("attempts", "outcome"));
let store = store();
RawAttempt {
state: "failed".to_string(),
outcome: Some(raw.clone()),
terminal_at: Some(timestamp_to_text(ts(2_000))),
..RawAttempt::default()
}
.insert(&store);
let error = store
.attempt(attempt_id())
.expect_err("an unterminated object is not an attempt outcome");
let rendered = error.to_string();
assert!(
!rendered.contains(planted),
"the secret must not appear in the message: {rendered}"
);
assert!(
!rendered.contains("ghs_"),
"and neither must a prefixed fragment of it, which is what a \
clipped echo would have produced and what `d1`'s shape-matching \
sink would then have failed to redact: {rendered}"
);
for len in 4..=planted.len() {
assert!(
!rendered.contains(&planted[..len]),
"no prefix of the secret may survive, and {:?} did: {rendered}",
&planted[..len]
);
}
assert!(
rendered.contains(&format!("{}-byte", raw.len())),
"the message must say how much is there: {rendered}"
);
assert!(
rendered.contains("stops parsing at line"),
"and where it stopped: {rendered}"
);
assert!(
rendered.contains(&attempt_id().to_string()),
"the error must name the row to fix: {rendered}"
);
let drifted = "ghs_9tokenish";
let raw = format!(r#"{{"outcome":"failed","reason":"{drifted}"}}"#);
serde_json::from_str::<serde_json::Value>(&raw).expect("this row is valid JSON");
let inner = serde_json::from_str::<AttemptOutcome>(&raw)
.expect_err("`ghs_9tokenish` is not a FailureReason");
assert_eq!(
(inner.line(), inner.column()),
(0, 0),
"the premise of this half of the test: serde has no position here"
);
assert!(
inner.to_string().contains(drifted),
"serde names the offending variant, so the message is not safe to \
forward: {inner}"
);
let drift_store = self::store();
RawAttempt {
state: "failed".to_string(),
outcome: Some(raw.clone()),
terminal_at: Some(timestamp_to_text(ts(2_000))),
..RawAttempt::default()
}
.insert(&drift_store);
let rendered = drift_store
.attempt(attempt_id())
.expect_err("an unknown reason variant is not an attempt outcome")
.to_string();
assert!(
!rendered.contains(drifted),
"the secret must not appear here either: {rendered}"
);
for len in 4..=drifted.len() {
assert!(
!rendered.contains(&drifted[..len]),
"no prefix of the secret may survive, and {:?} did: {rendered}",
&drifted[..len]
);
}
assert!(
rendered.contains(&format!("{}-byte", raw.len())),
"the message must say how much is there: {rendered}"
);
assert!(
rendered.contains(&attempt_id().to_string()),
"the error must name the row to fix: {rendered}"
);
assert!(
!rendered.contains("line 0")
&& !rendered.contains("column 0")
&& !rendered.contains("stops parsing at"),
"a position serde did not record must not be printed as one: \
{rendered}"
);
assert!(
rendered.contains("no position"),
"and the message has to say so, or the absence is indistinguishable \
from an omission: {rendered}"
);
}
#[test]
fn an_unknown_outcome_tag_keeps_the_position_serde_did_record() {
let raw = r#"{"outcome":"vanished"}"#;
let inner = serde_json::from_str::<AttemptOutcome>(raw)
.expect_err("`vanished` is not an attempt outcome");
assert_eq!(
inner.classify(),
serde_json::error::Category::Data,
"a `classify()`-based discriminator would send this down the \
positionless branch: {inner}"
);
assert_ne!(
inner.line(),
0,
"and it has a real position to lose, which is the whole finding: \
{inner}"
);
let store = store();
RawAttempt {
state: "failed".to_string(),
outcome: Some(raw.to_string()),
terminal_at: Some(timestamp_to_text(ts(2_000))),
..RawAttempt::default()
}
.insert(&store);
let rendered = store
.attempt(attempt_id())
.expect_err("an unknown outcome tag is not an attempt outcome")
.to_string();
assert!(
rendered.contains(&format!(
"stops parsing at line {}, column {}",
inner.line(),
inner.column()
)),
"the position serde did record must survive into the message: \
{rendered}"
);
assert!(
!rendered.contains("no position"),
"and must not be reported as absent: {rendered}"
);
assert!(
rendered.contains(&format!("{}-byte", raw.len())),
"the message must still say how much is there: {rendered}"
);
assert!(
!rendered.contains("vanished"),
"serde names the offending tag; this message must not: {rendered}"
);
}
#[test]
fn a_constrained_column_still_echoes_what_it_holds() {
assert!(!carries_free_form_text("attempts", "created_at"));
assert!(!carries_free_form_text("policies", "routing_labels"));
assert!(
Label::new("ghu_16CharsOfPaddingAndThenSomeMore1234567").is_ok(),
"a credential-shaped string is a valid Label, so the length and \
character rules are not what keeps `routing_labels` off the list"
);
let store = store();
RawAttempt {
created_at: "the third of never".to_string(),
..RawAttempt::default()
}
.insert(&store);
let error = store
.attempt(attempt_id())
.expect_err("`the third of never` is not RFC 3339");
assert!(
error.to_string().contains("the third of never"),
"a constrained column keeps its echo: {error}"
);
}
#[test]
fn the_journal_mode_is_read_back_rather_than_assumed() {
let memory = store();
assert_eq!(
memory.journal_mode(),
"memory",
"an in-memory database is exempt by construction"
);
assert!(
!memory.readers_do_not_block_writers(),
"there is no second reader of a private in-memory database, so the \
question does not arise for it"
);
let dir = tempfile::tempdir().expect("a temporary directory");
let path = dir.path().join("runner-manager.sqlite3");
let file = SqliteStore::open(&path).expect("opens");
let mode = file.journal_mode();
assert!(
matches!(mode, "wal" | "delete"),
"a file database is in WAL where the directory can host it and \
`delete` where it cannot; {mode} is neither and is a finding"
);
assert_eq!(
file.readers_do_not_block_writers(),
path.with_extension("sqlite3-wal").exists(),
"in {mode} mode the claim about readers and the write-ahead log on \
disk must be the same fact told twice"
);
assert!(
format!("{file:?}").contains(mode),
"an operator reading a support bundle should see the mode"
);
}
#[test]
fn every_column_lands_in_the_field_of_the_same_name() {
let store = store();
let configured_root = a_root("distinguishable-root");
RawHost {
host_capacity: 7,
refresh_interval_secs: 45,
created_at: timestamp_to_text(ts(1_234)),
display_name: "distinguishable-name".to_string(),
runner_root_override: Some(configured_root.as_str().to_string()),
..RawHost::default()
}
.insert(&store);
let host = store.host(host_id()).expect("loads").expect("present");
assert_eq!(host.id, host_id(), "hosts.id");
assert_eq!(host.display_name, "distinguishable-name");
assert_eq!(host.host_capacity.get(), 7, "hosts.host_capacity");
assert_eq!(
host.refresh_interval.as_secs(),
45,
"hosts.refresh_interval_secs"
);
assert_eq!(host.created_at, ts(1_234), "hosts.created_at");
assert_eq!(host.os, Os::Windows, "hosts.os");
assert_eq!(host.architecture, Arch::X64, "hosts.architecture");
assert_eq!(
host.service_start_mode,
StartMode::Boot,
"hosts.service_start_mode"
);
assert_eq!(
host.runner_root_override,
Some(configured_root),
"hosts.runner_root_override"
);
let persistent_root = a_root("distinguishable-workspace");
RawPolicy {
installation_id: 111,
revision: 222,
min_capacity: 3,
max_capacity: Some(9),
enabled: 0,
state: "pending".to_string(),
cache_policy: "discard_runner_package".to_string(),
target_slug: "owner/repo".to_string(),
workspace_mode: "persistent".to_string(),
workspace_path: Some(persistent_root.as_str().to_string()),
..RawPolicy::default()
}
.insert(&store);
let policy = store.policy(policy_id()).expect("loads").expect("present");
assert_eq!(policy.id, policy_id(), "policies.id");
assert_eq!(policy.host_id, host_id(), "policies.host_id");
assert_eq!(
policy.installation_id, 111,
"policies.installation_id must not come from policies.revision"
);
assert_eq!(
policy.revision(),
222,
"policies.revision must not come from policies.installation_id"
);
assert_eq!(policy.min_capacity(), 3, "policies.min_capacity");
assert_eq!(
policy.max_capacity().expect("autoscale").get(),
9,
"policies.max_capacity"
);
assert!(!policy.enabled(), "policies.enabled");
assert_eq!(policy.state(), PolicyState::Pending, "policies.state");
assert_eq!(
policy.cache_policy,
CachePolicy::DiscardRunnerPackage,
"policies.cache_policy"
);
assert_eq!(policy.target.slug(), "owner/repo", "policies.target_slug");
assert_eq!(
policy.target.scope(),
TargetScope::Repository,
"policies.target_scope"
);
assert_eq!(
policy
.routing_labels()
.expect("autoscale")
.host_label()
.as_str(),
"rm-home-win-x64",
"policies.routing_labels"
);
assert_eq!(
policy.workspace_policy().root(),
Some(&persistent_root),
"policies.workspace_path"
);
assert_eq!(
policy.workspace_policy().kind(),
WorkspaceKind::Persistent,
"policies.workspace_mode"
);
RawAttempt {
github_runner_id: Some(73),
process_id: Some(4_242),
state: "finished".to_string(),
outcome: Some(COMPLETED_JOB.to_string()),
runtime_path: "runtime/distinguishable".to_string(),
workspace_mode: "persistent".to_string(),
workspace_slot: Some(37),
created_at: timestamp_to_text(ts(1_000)),
last_state_change_at: timestamp_to_text(ts(2_000)),
terminal_at: Some(timestamp_to_text(ts(3_000))),
..RawAttempt::default()
}
.insert(&store);
let attempt = store
.attempt(attempt_id())
.expect("loads")
.expect("present");
assert_eq!(attempt.id, attempt_id(), "attempts.id");
assert_eq!(attempt.policy_id, policy_id(), "attempts.policy_id");
assert_eq!(
attempt.github_runner_id(),
Some(73),
"attempts.github_runner_id"
);
assert_eq!(attempt.process_id(), Some(4_242), "attempts.process_id");
assert_eq!(attempt.state(), AttemptState::Finished, "attempts.state");
assert_eq!(
attempt.outcome(),
Some(&AttemptOutcome::CompletedJob),
"attempts.outcome"
);
assert_eq!(
attempt.runtime_path(),
Path::new("runtime/distinguishable"),
"attempts.runtime_path"
);
assert_eq!(
attempt.created_at,
ts(1_000),
"attempts.created_at must not come from attempts.last_state_change_at"
);
assert_eq!(
attempt.last_state_change_at(),
ts(2_000),
"attempts.last_state_change_at must not come from attempts.created_at; \
every recovery timeout is measured from it"
);
assert_eq!(
attempt.terminal_at(),
Some(ts(3_000)),
"attempts.terminal_at"
);
assert_eq!(
attempt.workspace().slot_number(),
Some(37),
"attempts.workspace_slot"
);
assert_eq!(
attempt.workspace().kind(),
WorkspaceKind::Persistent,
"attempts.workspace_mode"
);
}
#[test]
fn a_hand_corrupted_policy_shape_is_rejected_on_load() {
let store = store();
RawPolicy {
routing_labels: Some(LABELS_JSON.to_string()),
max_capacity: None,
..RawPolicy::default()
}
.insert(&store);
assert!(
matches!(
store.policy(policy_id()),
Err(StoreError::CorruptPolicy {
source: PolicyError::AutoscaleWithoutMaxCapacity,
..
})
),
"labels without a ceiling could oversubscribe the host"
);
RawPolicy {
routing_labels: None,
max_capacity: Some(2),
..RawPolicy::default()
}
.insert(&store);
assert!(matches!(
store.policy(policy_id()),
Err(StoreError::CorruptPolicy {
source: PolicyError::AutoscaleWithoutRoutingLabels,
..
})
));
RawPolicy {
routing_labels: None,
max_capacity: None,
min_capacity: 1,
..RawPolicy::default()
}
.insert(&store);
assert!(matches!(
store.policy(policy_id()),
Err(StoreError::CorruptPolicy {
source: PolicyError::MonitorOnlyWithMinCapacity { min: 1 },
..
})
));
RawPolicy {
min_capacity: 3,
max_capacity: Some(2),
..RawPolicy::default()
}
.insert(&store);
assert!(
matches!(
store.policy(policy_id()),
Err(StoreError::CorruptPolicy {
source: PolicyError::InvertedCapacityRange { min: 3, max: 2 },
..
})
),
"an inverted range makes clamp(demand, min, max) panic, so it must \
not survive a load"
);
RawPolicy {
target_scope: "organization".to_string(),
target_slug: "o/r".to_string(),
..RawPolicy::default()
}
.insert(&store);
assert!(
matches!(
store.policy(policy_id()),
Err(StoreError::CorruptPolicy {
source: PolicyError::Invalid(ValidationError::IllegalCharacter {
found: '/',
..
}),
..
})
),
"the target is rebuilt through the real constructor, so GitHub's \
naming rules run again on load"
);
for (label, raw) in [
(
"a zero ceiling",
RawPolicy {
max_capacity: Some(0),
..RawPolicy::default()
},
),
(
"a third spelling of enabled",
RawPolicy {
enabled: 7,
..RawPolicy::default()
},
),
(
"an unrecognised state",
RawPolicy {
state: "retired".to_string(),
..RawPolicy::default()
},
),
(
"a negative revision",
RawPolicy {
revision: -1,
..RawPolicy::default()
},
),
(
"a routing label carrying the separator the runner splits on",
RawPolicy {
routing_labels: Some(r#"{"host_label":"bad,label"}"#.to_string()),
..RawPolicy::default()
},
),
] {
raw.insert(&store);
assert!(
matches!(
store.policy(policy_id()),
Err(StoreError::CorruptColumn { .. })
),
"{label} must be reported as a corrupt column"
);
}
}
#[test]
fn a_hand_corrupted_host_row_is_rejected_on_load() {
let store = store();
RawHost {
refresh_interval_secs: 1,
..RawHost::default()
}
.insert(&store);
assert!(
matches!(
store.host(host_id()),
Err(StoreError::CorruptHost {
source: ValidationError::BelowFloor {
min: 30,
actual: 1,
..
},
..
})
),
"a hand-edited row must not make this host poll every second; the \
floor is a rate-budget constraint"
);
RawHost {
display_name: " ".to_string(),
..RawHost::default()
}
.insert(&store);
assert!(matches!(
store.host(host_id()),
Err(StoreError::CorruptHost {
source: ValidationError::Empty { .. },
..
})
));
for (label, raw) in [
(
"zero capacity",
RawHost {
host_capacity: 0,
..RawHost::default()
},
),
(
"an unsupported operating system",
RawHost {
os: "plan9".to_string(),
..RawHost::default()
},
),
(
"a malformed created_at",
RawHost {
created_at: "yesterday".to_string(),
..RawHost::default()
},
),
] {
raw.insert(&store);
assert!(
matches!(store.host(host_id()), Err(StoreError::CorruptColumn { .. })),
"{label} must be reported as a corrupt column"
);
}
}
#[test]
fn a_hand_corrupted_attempt_row_is_rejected_on_load() {
let store = store();
for (label, raw) in [
(
"terminal with no outcome",
RawAttempt {
state: "finished".to_string(),
outcome: None,
terminal_at: Some(timestamp_to_text(ts(2_000))),
..RawAttempt::default()
},
),
(
"non-terminal carrying an outcome",
RawAttempt {
state: "busy".to_string(),
outcome: Some(COMPLETED_JOB.to_string()),
..RawAttempt::default()
},
),
(
"a failed attempt claiming it ran a job",
RawAttempt {
state: "failed".to_string(),
outcome: Some(COMPLETED_JOB.to_string()),
terminal_at: Some(timestamp_to_text(ts(2_000))),
..RawAttempt::default()
},
),
(
"terminal with no terminal_at",
RawAttempt {
state: "finished".to_string(),
outcome: Some(COMPLETED_JOB.to_string()),
terminal_at: None,
..RawAttempt::default()
},
),
] {
raw.insert(&store);
assert!(
matches!(
store.attempt(attempt_id()),
Err(StoreError::CorruptAttempt { .. })
),
"{label} must not load"
);
}
for (label, raw) in [
(
"a negative process id",
RawAttempt {
process_id: Some(-1),
..RawAttempt::default()
},
),
(
"an unrecognised state",
RawAttempt {
state: "wedged".to_string(),
..RawAttempt::default()
},
),
(
"an outcome that is not one",
RawAttempt {
state: "finished".to_string(),
outcome: Some(r#"{"outcome":"went_home"}"#.to_string()),
terminal_at: Some(timestamp_to_text(ts(2_000))),
..RawAttempt::default()
},
),
(
"a malformed created_at",
RawAttempt {
created_at: "yesterday".to_string(),
..RawAttempt::default()
},
),
] {
raw.insert(&store);
assert!(
matches!(
store.attempt(attempt_id()),
Err(StoreError::CorruptColumn { .. })
),
"{label} must be reported as a corrupt column"
);
}
}
#[test]
fn a_hand_corrupted_host_runner_root_is_rejected_on_load() {
let store = store();
for (label, raw) in [
("a UNC share", r"\\nas\builds"),
("a relative path", "runners"),
(
"a traversal component",
if cfg!(windows) {
r"X:\rman\..\elsewhere"
} else {
"/srv/rman/../elsewhere"
},
),
(
"a bare filesystem root",
if cfg!(windows) { r"X:\" } else { "/" },
),
(
"a path from the other platform",
if cfg!(windows) {
"/srv/rman"
} else {
r"X:\rman"
},
),
] {
RawHost {
runner_root_override: Some(raw.to_string()),
..RawHost::default()
}
.insert(&store);
let error = store
.host(host_id())
.expect_err(&format!("{label} must not load"));
assert!(
matches!(error, StoreError::CorruptHostWorkspace { id, .. } if id == host_id()),
"{label} must be reported against the host row, got {error:?}"
);
assert!(
!error.is_conflict(),
"{label} is corrupt state, not a concurrency conflict"
);
}
let configured = a_root("rman");
RawHost {
runner_root_override: Some(configured.as_str().to_string()),
..RawHost::default()
}
.insert(&store);
let host = store.host(host_id()).expect("loads").expect("present");
assert_eq!(host.runner_root_override, Some(configured));
assert!(host.has_configured_runner_root());
}
#[test]
fn a_hand_corrupted_policy_workspace_is_rejected_on_load() {
let store = store();
let root = a_root("workspaces");
for (label, raw) in [
(
"persistent without a path",
RawPolicy {
workspace_mode: "persistent".to_string(),
workspace_path: None,
..RawPolicy::default()
},
),
(
"ephemeral with a stale path",
RawPolicy {
workspace_mode: "ephemeral".to_string(),
workspace_path: Some(root.as_str().to_string()),
..RawPolicy::default()
},
),
(
"an organization policy claiming to retain a workspace",
RawPolicy {
target_scope: "organization".to_string(),
target_slug: "tap-top-fun".to_string(),
workspace_mode: "persistent".to_string(),
workspace_path: Some(root.as_str().to_string()),
..RawPolicy::default()
},
),
(
"a persistent root that is not a storable local path",
RawPolicy {
workspace_mode: "persistent".to_string(),
workspace_path: Some(r"\\nas\builds".to_string()),
..RawPolicy::default()
},
),
] {
raw.insert(&store);
let error = store
.policy(policy_id())
.expect_err(&format!("{label} must not load"));
assert!(
matches!(error, StoreError::CorruptPolicy { id, .. } if id == policy_id()),
"{label} must be reported against the policy row, got {error:?}"
);
}
RawPolicy {
workspace_mode: "sticky".to_string(),
..RawPolicy::default()
}
.insert(&store);
assert!(
matches!(
store.policy(policy_id()),
Err(StoreError::CorruptColumn {
table: "policies",
column: "workspace_mode",
..
})
),
"an unknown workspace mode must fail closed"
);
RawPolicy {
workspace_mode: "persistent".to_string(),
workspace_path: Some(root.as_str().to_string()),
..RawPolicy::default()
}
.insert(&store);
let policy = store.policy(policy_id()).expect("loads").expect("present");
assert_eq!(policy.workspace_policy().root(), Some(&root));
assert!(policy.workspace_policy().retains_job_workspace());
}
#[test]
fn a_hand_corrupted_attempt_workspace_is_rejected_on_load() {
let store = store();
for (label, raw) in [
(
"persistent without a slot",
RawAttempt {
workspace_mode: "persistent".to_string(),
workspace_slot: None,
..RawAttempt::default()
},
),
(
"ephemeral holding a slot",
RawAttempt {
workspace_mode: "ephemeral".to_string(),
workspace_slot: Some(1),
..RawAttempt::default()
},
),
(
"slot zero, which names no directory",
RawAttempt {
workspace_mode: "persistent".to_string(),
workspace_slot: Some(0),
..RawAttempt::default()
},
),
] {
raw.insert(&store);
assert!(
matches!(
store.attempt(attempt_id()),
Err(StoreError::CorruptAttempt { .. })
),
"{label} must not load"
);
}
for (label, raw) in [
(
"an unrecognised mode",
RawAttempt {
workspace_mode: "sticky".to_string(),
..RawAttempt::default()
},
),
(
"a slot above u16",
RawAttempt {
workspace_mode: "persistent".to_string(),
workspace_slot: Some(70_000),
..RawAttempt::default()
},
),
] {
raw.insert(&store);
assert!(
matches!(
store.attempt(attempt_id()),
Err(StoreError::CorruptColumn {
table: "attempts",
..
})
),
"{label} must be reported as a corrupt column"
);
}
RawAttempt {
workspace_mode: "persistent".to_string(),
workspace_slot: Some(2),
..RawAttempt::default()
}
.insert(&store);
let attempt = store
.attempt(attempt_id())
.expect("loads")
.expect("present");
assert_eq!(
attempt.workspace(),
AttemptWorkspace::persistent_slot(NonZeroU16::new(2).expect("non-zero"))
);
assert!(attempt.holds_slot_lease());
}
fn outcome_for(state: AttemptState) -> Option<AttemptOutcome> {
match state {
AttemptState::Failed => Some(AttemptOutcome::failed(
FailureReason::ProcessExitedUnexpectedly,
)),
AttemptState::Orphaned => Some(AttemptOutcome::Orphaned),
AttemptState::Finished | AttemptState::Cleaned => Some(AttemptOutcome::CompletedJob),
_ => None,
}
}
fn attempt_in_state(attempt: &RunnerAttempt, state: AttemptState) -> RunnerAttempt {
let mut fields = attempt.to_persisted();
fields.state = state;
fields.outcome = outcome_for(state);
fields.terminal_at = state.is_terminal().then(|| ts(2_000));
fields.last_state_change_at = ts(2_000);
RunnerAttempt::from_persisted(fields).expect("a state the domain accepts")
}
fn persistent_attempt(id: u128, slot: u16, state: AttemptState) -> RunnerAttempt {
let attempt = RunnerAttempt::allocate_in(
AttemptId::from_u128(id),
policy_id(),
format!("root/s{slot}"),
AttemptWorkspace::persistent_slot(NonZeroU16::new(slot).expect("positive")),
ts(1_000),
);
if state == AttemptState::Allocated {
return attempt;
}
attempt_in_state(&attempt, state)
}
#[test]
fn one_slot_is_leased_to_at_most_one_uncleaned_attempt() {
let store = store();
let first = persistent_attempt(0x501, 1, AttemptState::Idle);
store.record_attempt(&first).expect("the first lease");
let second = persistent_attempt(0x502, 1, AttemptState::Allocated);
let error = store
.record_attempt(&second)
.expect_err("a second uncleaned attempt must not take a leased slot");
assert!(
matches!(
error,
StoreError::SlotAlreadyLeased { policy, slot }
if policy == policy_id() && slot == 1
),
"expected SlotAlreadyLeased, got {error:?}"
);
assert!(
error.is_conflict(),
"an allocator that lost a slot picks another one; this is not an \
I/O failure"
);
assert!(
store.attempt(second.id).expect("loads").is_none(),
"the refused insert must not be half-applied"
);
let leases = store.slot_leases_for_policy(policy_id()).expect("loads");
assert_eq!(leases.len(), 1);
assert_eq!(leases[0].id, first.id);
store
.record_attempt(&persistent_attempt(0x503, 2, AttemptState::Allocated))
.expect("slot 2 is not leased");
let other_policy = RunnerAttempt::allocate_in(
AttemptId::from_u128(0x504),
PolicyId::from_u128(0x11),
"root/s1",
AttemptWorkspace::persistent_slot(NonZeroU16::new(1).expect("positive")),
ts(1_000),
);
store
.record_attempt(&other_policy)
.expect("another policy's s1 is a different slot");
}
#[test]
fn a_cleaned_historical_attempt_releases_its_slot_for_reuse() {
let store = store();
let first = persistent_attempt(0x511, 1, AttemptState::Cleaned);
store.record_attempt(&first).expect("a cleaned lease");
assert!(!first.holds_slot_lease());
assert!(
store
.slot_leases_for_policy(policy_id())
.expect("loads")
.is_empty(),
"a cleaned attempt holds no lease"
);
let second = persistent_attempt(0x512, 1, AttemptState::Allocated);
store
.record_attempt(&second)
.expect("a cleaned row must not block reuse of its slot");
assert_eq!(
store.attempts_for_policy(policy_id()).expect("loads").len(),
2,
"the historical row is kept, not deleted, when the slot is reused"
);
assert!(matches!(
store.record_attempt(&persistent_attempt(0x513, 1, AttemptState::Allocated)),
Err(StoreError::SlotAlreadyLeased { slot: 1, .. })
));
}
#[test]
fn a_terminal_attempt_awaiting_cleanup_still_holds_its_slot() {
let store = store();
let failed_cleanup = persistent_attempt(0x521, 1, AttemptState::Finished);
store.record_attempt(&failed_cleanup).expect("journalled");
assert!(
store
.active_attempts_for_policy(policy_id())
.expect("loads")
.is_empty(),
"a terminal attempt occupies no capacity slot"
);
let leases = store.slot_leases_for_policy(policy_id()).expect("loads");
assert_eq!(leases.len(), 1, "but it does still hold its lease");
assert_eq!(leases[0].workspace().slot_number(), Some(1));
assert!(matches!(
store.record_attempt(&persistent_attempt(0x522, 1, AttemptState::Allocated)),
Err(StoreError::SlotAlreadyLeased { slot: 1, .. })
));
}
#[test]
fn journalling_the_same_attempt_again_cannot_move_its_lease() {
let store = store();
let allocated = persistent_attempt(0x531, 1, AttemptState::Allocated);
store.record_attempt(&allocated).expect("journalled");
let mut fields = allocated.to_persisted();
fields.state = AttemptState::Idle;
fields.last_state_change_at = ts(2_000);
fields.workspace_slot = Some(9);
let moved = RunnerAttempt::from_persisted(fields).expect("a legal attempt in isolation");
store
.record_attempt(&moved)
.expect("the state change lands");
let stored = store
.attempt(allocated.id)
.expect("loads")
.expect("present");
assert_eq!(stored.state(), AttemptState::Idle, "the state did move");
assert_eq!(
stored.workspace().slot_number(),
Some(1),
"the slot journalled at allocation is the one that stands"
);
}
#[test]
fn the_attempt_set_predicates_follow_the_domain() {
let store = store();
for (index, state) in AttemptState::ALL.into_iter().enumerate() {
let attempt = RunnerAttempt::allocate(
AttemptId::from_u128(0x600 + index as u128),
policy_id(),
format!("runtime/{state}"),
ts(1_000),
);
store
.record_attempt(&attempt_in_state(&attempt, state))
.expect("journalled");
}
let expected_active = AttemptState::ALL
.into_iter()
.filter(|state| state.counts_against_capacity())
.count();
let expected_uncleaned = AttemptState::ALL
.into_iter()
.filter(|state| *state != AttemptState::Cleaned)
.count();
assert_ne!(
expected_active, expected_uncleaned,
"if these were equal the two fences would be the same fence and this \
test would prove nothing"
);
assert_eq!(
store
.active_attempts_for_policy(policy_id())
.expect("loads")
.len(),
expected_active
);
assert_eq!(
store
.uncleaned_attempts_for_policy(policy_id())
.expect("loads")
.len(),
expected_uncleaned
);
assert_eq!(
store.uncleaned_ephemeral_attempts().expect("loads").len(),
expected_uncleaned,
"every attempt here is ephemeral, so the host-wide set is the same \
size as the per-policy uncleaned one"
);
assert!(
store
.slot_leases_for_policy(policy_id())
.expect("loads")
.is_empty(),
"and none of them is a slot lease"
);
}
#[test]
fn the_host_wide_ephemeral_set_counts_an_attempt_that_outlived_its_policy() {
let store = store();
let orphan = RunnerAttempt::allocate(
AttemptId::from_u128(0x701),
PolicyId::from_u128(0xdead),
"runtime/orphan",
ts(1_000),
);
store.record_attempt(&orphan).expect("journalled");
store
.record_attempt(&persistent_attempt(0x702, 1, AttemptState::Allocated))
.expect("journalled");
let ephemeral = store.uncleaned_ephemeral_attempts().expect("loads");
assert_eq!(ephemeral.len(), 1, "the persistent attempt is not counted");
assert_eq!(ephemeral[0].id, orphan.id);
}
fn a_stored_host(store: &SqliteStore) -> Host {
let host = Host::new(
host_id(),
"home-pc",
Os::Windows,
Arch::X64,
NonZeroU16::new(2).expect("non-zero"),
ts(1_000),
)
.expect("valid");
store.put_host(&host).expect("stored");
host
}
#[test]
fn the_host_root_mutation_writes_only_its_own_column() {
let store = store();
let read = a_stored_host(&store);
let mut concurrent = read.clone();
concurrent.host_capacity = NonZeroU16::new(7).expect("non-zero");
concurrent.service_start_mode = StartMode::Login;
store.put_host(&concurrent).expect("the other writer wins");
let configured = a_root("rman");
store
.set_runner_root_override(host_id(), None, Some(&configured), 0)
.expect("the root moves");
let stored = store.host(host_id()).expect("loads").expect("present");
assert_eq!(stored.runner_root_override, Some(configured));
assert_eq!(
stored.host_capacity.get(),
7,
"a whole-record write built from the stale read would have rolled \
this back to 2"
);
assert_eq!(stored.service_start_mode, StartMode::Login);
}
#[test]
fn the_host_root_mutation_refuses_a_changed_expected_override() {
let store = store();
a_stored_host(&store);
let first = a_root("rman");
let second = a_root("elsewhere");
store
.set_runner_root_override(host_id(), None, Some(&first), 0)
.expect("the first mutation");
let error = store
.set_runner_root_override(host_id(), None, Some(&second), 0)
.expect_err("a stale expected override must be refused");
assert!(
matches!(&error, StoreError::RunnerRootChanged { id, .. } if *id == host_id()),
"expected RunnerRootChanged, got {error:?}"
);
assert!(error.is_conflict());
let message = error.to_string();
assert!(
message.contains("the platform default") && message.contains(first.as_str()),
"the message must name both sides so the operator can re-read: \
{message}"
);
assert_eq!(
store
.host(host_id())
.expect("loads")
.expect("present")
.runner_root_override,
Some(first.clone()),
"nothing was written"
);
store
.set_runner_root_override(host_id(), Some(&first), None, 0)
.expect("reset to the platform default");
assert_eq!(
store
.host(host_id())
.expect("loads")
.expect("present")
.runner_root_override,
None
);
}
#[test]
fn the_host_root_mutation_refuses_a_changed_uncleaned_ephemeral_count() {
let store = store();
a_stored_host(&store);
let configured = a_root("rman");
let mut attempt = RunnerAttempt::allocate(
attempt_id(),
policy_id(),
"runtime/policy/attempt",
ts(1_000),
);
store.record_attempt(&attempt).expect("journalled");
let error = store
.set_runner_root_override(host_id(), None, Some(&configured), 0)
.expect_err("an attempt appeared after the operator counted zero");
assert!(
matches!(
&error,
StoreError::UncleanedCountChanged { expected: 0, found: 1, subject }
if subject.contains(&host_id().to_string())
),
"expected UncleanedCountChanged naming the host, got {error:?}"
);
assert!(error.is_conflict());
store
.set_runner_root_override(host_id(), None, Some(&configured), 1)
.expect("the confirmed count matches");
attempt = attempt_in_state(&attempt, AttemptState::Finished);
store.record_attempt(&attempt).expect("journalled");
assert!(
store
.active_attempts_for_policy(policy_id())
.expect("loads")
.is_empty()
);
assert!(matches!(
store.set_runner_root_override(host_id(), Some(&configured), None, 0),
Err(StoreError::UncleanedCountChanged { found: 1, .. })
));
}
#[test]
fn the_host_root_mutation_reports_a_missing_host_rather_than_a_conflict() {
let store = store();
let error = store
.set_runner_root_override(host_id(), None, Some(&a_root("rman")), 0)
.expect_err("there is no such host");
assert!(
matches!(error, StoreError::NotFound { what: "host", .. }),
"expected NotFound, got {error:?}"
);
assert!(!error.is_conflict());
}
fn a_stored_policy(store: &SqliteStore) -> ScalePolicy {
let policy = ScalePolicy::new(
policy_id(),
ScaleTarget::repository("o/r").expect("valid"),
1,
host_id(),
PolicyMode::autoscale(
RoutingLabels::from_host_label(Label::new("rm-home-win-x64").expect("valid")),
0,
NonZeroU16::new(2).expect("non-zero"),
)
.expect("valid"),
CachePolicy::default(),
);
store.insert_policy(&policy).expect("inserted");
policy
}
#[test]
fn the_policy_workspace_mutation_confirms_revision_and_uncleaned_count() {
let store = store();
let mut policy = a_stored_policy(&store);
let read_revision = policy.revision();
let root = a_root("workspaces");
policy
.set_workspace_policy(
WorkspacePolicy::persistent(root.clone(), TargetScope::Repository)
.expect("a repository may"),
)
.expect("a repository may");
assert!(matches!(
store.update_policy_confirming_uncleaned_count(&policy, read_revision + 7, 0),
Err(StoreError::StaleRevision { .. })
));
let attempt =
RunnerAttempt::allocate(attempt_id(), policy_id(), "runtime/attempt", ts(1_000));
store
.record_attempt(&attempt_in_state(&attempt, AttemptState::Failed))
.expect("journalled");
let error = store
.update_policy_confirming_uncleaned_count(&policy, read_revision, 0)
.expect_err("an unresolved attempt appeared");
assert!(
matches!(
&error,
StoreError::UncleanedCountChanged { expected: 0, found: 1, subject }
if subject.contains(&policy_id().to_string())
),
"expected UncleanedCountChanged naming the policy, got {error:?}"
);
assert!(error.is_conflict());
assert_eq!(
store
.policy(policy_id())
.expect("loads")
.expect("present")
.workspace_policy(),
&WorkspacePolicy::Ephemeral,
"nothing was written"
);
assert!(
store
.update_policy_confirming_active_count(&policy, read_revision, 0)
.is_ok(),
"an unresolved attempt is invisible to the active-count guard"
);
}
#[test]
fn a_workspace_policy_survives_a_guarded_write_and_a_reload() {
let store = store();
let mut policy = a_stored_policy(&store);
let root = a_root("workspaces");
let read_revision = policy.revision();
policy
.set_workspace_policy(
WorkspacePolicy::persistent(root.clone(), TargetScope::Repository)
.expect("a repository may"),
)
.expect("a repository may");
store
.update_policy_confirming_uncleaned_count(&policy, read_revision, 0)
.expect("no attempt stands in the way");
let stored = store.policy(policy_id()).expect("loads").expect("present");
assert_eq!(&stored, &policy);
assert_eq!(stored.workspace_policy().root(), Some(&root));
let read_revision = stored.revision();
let mut back = stored;
back.set_workspace_policy(WorkspacePolicy::Ephemeral)
.expect("always permitted");
store
.update_policy_confirming_uncleaned_count(&back, read_revision, 0)
.expect("no attempt stands in the way");
let stored = store.policy(policy_id()).expect("loads").expect("present");
assert_eq!(stored.workspace_policy(), &WorkspacePolicy::Ephemeral);
let raw: Option<String> = store
.lock()
.query_row(
"SELECT workspace_path FROM policies WHERE id = :id",
named_params! { ":id": POLICY_UUID },
|row| row.get(0),
)
.expect("readable");
assert_eq!(raw, None, "the column is cleared, not merely ignored");
}
#[test]
fn active_count_guard_fences_zero_to_one_and_one_to_zero_attempt_writes() {
for starts_active in [false, true] {
ATTEMPT_WRITE_BLOCKED.store(false, Ordering::Release);
let directory = tempfile::TempDir::new().expect("temporary database directory");
let path = directory.path().join("state.sqlite3");
let updater = SqliteStore::open(&path).expect("update connection");
let writer = Arc::new(SqliteStore::open(&path).expect("attempt connection"));
writer
.lock()
.busy_handler(Some(mark_attempt_write_blocked))
.expect("test busy observer");
let mut policy = ScalePolicy::new(
policy_id(),
ScaleTarget::repository("o/r").expect("valid"),
1,
host_id(),
PolicyMode::autoscale(
RoutingLabels::from_host_label(Label::new("rm-home-win-x64").expect("valid")),
0,
NonZeroU16::new(2).expect("non-zero"),
)
.expect("valid"),
CachePolicy::default(),
);
policy.activate().expect("active policy");
updater.insert_policy(&policy).expect("inserted");
let attempt = RunnerAttempt::allocate(attempt_id(), policy.id, "runner", ts(1_000));
if starts_active {
updater.record_attempt(&attempt).expect("active attempt");
}
let expected_active = u16::from(starts_active);
let mut disabled = policy.clone();
disabled.request_disable().expect("disable requested");
if !starts_active {
disabled
.drain_completed(0)
.expect("zero drains immediately");
}
let (begin_tx, write_attempt) = mpsc::channel();
let (finished_tx, finished_rx) = mpsc::channel();
let writer_thread = Arc::clone(&writer);
let attempt_for_thread = attempt.clone();
let handle = std::thread::spawn(move || {
write_attempt.recv().expect("transaction began");
if starts_active {
writer_thread
.remove_attempt(attempt_for_thread.id)
.expect("completion write");
} else {
writer_thread
.record_attempt(&attempt_for_thread)
.expect("allocation write");
}
finished_tx.send(()).expect("completion observed");
});
updater
.update_policy_confirming_active_count_with(
&disabled,
policy.revision(),
expected_active,
|| {
begin_tx.send(()).expect("release attempt writer");
let deadline = Instant::now() + Duration::from_secs(5);
while !ATTEMPT_WRITE_BLOCKED.load(Ordering::Acquire) {
assert!(
Instant::now() < deadline,
"attempt writer never reached SQLite's allocation fence"
);
std::thread::yield_now();
}
assert!(
matches!(finished_rx.try_recv(), Err(mpsc::TryRecvError::Empty)),
"attempt mutation crossed the policy transaction"
);
},
)
.expect("confirmed count commits while the attempt writer is fenced");
handle
.join()
.expect("attempt writer completed after commit");
finished_rx
.recv()
.expect("attempt mutation eventually commits");
let stored = updater.policy(policy.id).unwrap().unwrap();
assert!(!stored.enabled());
assert_eq!(
updater.attempt(attempt.id).unwrap().is_some(),
!starts_active,
"the attempt mutation must occur only after policy persistence"
);
}
}
#[test]
fn a_stale_revision_write_is_rejected_and_is_not_an_io_error() {
let store = store();
let mut policy = ScalePolicy::new(
policy_id(),
ScaleTarget::repository("o/r").expect("valid"),
1,
host_id(),
PolicyMode::autoscale(
RoutingLabels::from_host_label(Label::new("rm-home-win-x64").expect("valid")),
0,
NonZeroU16::new(2).expect("non-zero"),
)
.expect("valid"),
CachePolicy::default(),
);
store.insert_policy(&policy).expect("inserted");
assert_eq!(policy.revision(), 0);
let mut tui_copy = store.policy(policy_id()).expect("loads").expect("present");
tui_copy.activate().expect("a pending policy activates");
store
.update_policy(&tui_copy, 0)
.expect("the first write wins");
assert_eq!(
store
.policy(policy_id())
.expect("loads")
.expect("present")
.revision(),
1
);
policy
.set_max_capacity(NonZeroU16::new(5).expect("non-zero"))
.expect("autoscale");
let error = store
.update_policy(&policy, 0)
.expect_err("the second write must be rejected");
assert!(
matches!(
error,
StoreError::StaleRevision {
expected: 0,
found: 1,
..
}
),
"expected a stale-revision rejection, got {error:?}"
);
assert!(
error.is_conflict(),
"the caller must be able to tell a lost race from an I/O failure"
);
let stored = store.policy(policy_id()).expect("loads").expect("present");
assert_eq!(stored.max_capacity().expect("autoscale").get(), 2);
assert!(stored.enabled());
let mut fresh = stored;
fresh
.set_max_capacity(NonZeroU16::new(5).expect("non-zero"))
.expect("autoscale");
store.update_policy(&fresh, 1).expect("the retry wins");
assert_eq!(
store
.policy(policy_id())
.expect("loads")
.expect("present")
.max_capacity()
.expect("autoscale")
.get(),
5
);
}
#[test]
fn removing_a_policy_takes_the_same_revision_check() {
let store = store();
RawPolicy {
revision: 4,
..RawPolicy::default()
}
.insert(&store);
let error = store
.remove_policy(policy_id(), 3)
.expect_err("a stale delete must be rejected");
assert!(error.is_conflict(), "got {error:?}");
assert!(
store.policy(policy_id()).expect("loads").is_some(),
"a rejected delete must not delete"
);
store
.remove_policy(policy_id(), 4)
.expect("the current revision deletes");
assert!(store.policy(policy_id()).expect("loads").is_none());
}
#[test]
fn a_revision_guarded_write_to_a_missing_row_is_not_found_not_a_conflict() {
let store = store();
let error = store
.remove_policy(policy_id(), 0)
.expect_err("there is nothing to delete");
assert!(
matches!(error, StoreError::NotFound { what: "policy", .. }),
"a missing row is a different problem from a lost race: {error:?}"
);
assert!(!error.is_conflict());
}
#[test]
fn inserting_a_policy_twice_is_reported_as_already_existing() {
let store = store();
RawPolicy::default().insert(&store);
let policy = store.policy(policy_id()).expect("loads").expect("present");
let error = store.insert_policy(&policy).expect_err("the id is taken");
assert!(
matches!(error, StoreError::AlreadyExists { what: "policy", .. }),
"got {error:?}"
);
}
#[test]
fn created_at_is_never_rewritten_by_a_later_journal_write() {
let store = store();
let allocated = RunnerAttempt::allocate(attempt_id(), policy_id(), "runtime/x", ts(1_000));
store.record_attempt(&allocated).expect("journalled");
let rewritten = RunnerAttempt::from_persisted(PersistedAttempt {
created_at: ts(5_000),
last_state_change_at: ts(5_000),
..allocated.to_persisted()
})
.expect("a legal attempt");
store.record_attempt(&rewritten).expect("journalled");
let stored = store
.attempt(attempt_id())
.expect("loads")
.expect("present");
assert_eq!(stored.created_at, ts(1_000), "created_at never moves");
assert_eq!(
stored.last_state_change_at(),
ts(5_000),
"every other column is updated in place"
);
}
#[test]
fn a_backwards_clock_is_clamped_on_write_and_on_load() {
let store = store();
let mut attempt =
RunnerAttempt::allocate(attempt_id(), policy_id(), "runtime/x", ts(1_000));
attempt
.jit_received(ts(900))
.expect("the domain accepts a backwards `now` with no ordering check");
assert!(
matches!(
RunnerAttempt::from_persisted(attempt.to_persisted()),
Err(AttemptError::TimestampsOutOfOrder {
field: "last_state_change_at",
..
})
),
"if the domain ever accepts this, `normalise` is dead code and should go"
);
store.record_attempt(&attempt).expect("journalled");
assert_eq!(store.clock_skew_repairs(), 1);
{
let conn = store.lock();
let raw: String = conn
.query_row(
"SELECT last_state_change_at FROM attempts WHERE id = ?1",
[ATTEMPT_UUID],
|row| row.get(0),
)
.expect("readable");
assert_eq!(
raw,
timestamp_to_text(ts(1_000)),
"the write path stores a row that can be read back, not one that \
poisons the journal"
);
}
let back = store
.attempt(attempt_id())
.expect("loads")
.expect("present");
assert_eq!(back.last_state_change_at(), ts(1_000));
assert_eq!(
store.clock_skew_repairs(),
1,
"the stored row is already sound, so loading it repairs nothing"
);
RawAttempt {
created_at: timestamp_to_text(ts(1_000)),
last_state_change_at: timestamp_to_text(ts(400)),
state: "finished".to_string(),
outcome: Some(COMPLETED_JOB.to_string()),
terminal_at: Some(timestamp_to_text(ts(500))),
..RawAttempt::default()
}
.insert(&store);
let repaired = store
.attempt(attempt_id())
.expect("loads")
.expect("present");
assert_eq!(repaired.last_state_change_at(), ts(1_000));
assert_eq!(repaired.terminal_at(), Some(ts(1_000)));
assert_eq!(
store.clock_skew_repairs(),
3,
"both out-of-order timestamps are counted, and each is logged at warn"
);
}
#[test]
fn a_runtime_path_that_is_not_utf8_is_refused_rather_than_mangled() {
#[cfg(windows)]
let bad: PathBuf = {
use std::os::windows::ffi::OsStringExt as _;
std::ffi::OsString::from_wide(&[0x0072, 0xD800]).into()
};
#[cfg(not(windows))]
let bad: PathBuf = {
use std::os::unix::ffi::OsStringExt as _;
std::ffi::OsString::from_vec(vec![b'r', 0xFF]).into()
};
assert!(bad.to_str().is_none(), "the fixture path must be non-UTF-8");
let store = store();
let attempt = RunnerAttempt::allocate(attempt_id(), policy_id(), bad, ts(1_000));
let error = store
.record_attempt(&attempt)
.expect_err("a path that cannot round-trip must not be stored");
assert!(
matches!(error, StoreError::UnrepresentablePath { .. }),
"got {error:?}"
);
}
#[test]
fn an_integer_too_large_for_a_sqlite_integer_is_refused_rather_than_truncated() {
let store = store();
let mut attempt =
RunnerAttempt::allocate(attempt_id(), policy_id(), "runtime/x", ts(1_000));
attempt.jit_received(ts(1_001)).expect("a legal transition");
attempt
.started(4_242, ts(1_002))
.expect("a legal transition");
attempt
.registered_idle(u64::MAX, ts(1_003))
.expect("the domain accepts any u64 as a runner id");
let error = store
.record_attempt(&attempt)
.expect_err("a value SQLite cannot hold must not be silently truncated");
assert!(
matches!(
error,
StoreError::UnrepresentableInteger {
what: "attempts.github_runner_id",
value: u64::MAX,
}
),
"got {error:?}"
);
let biggest = u64::try_from(i64::MAX).expect("i64::MAX is a valid u64");
let mut ok = RunnerAttempt::allocate(
AttemptId::from_u128(0x0000_0101),
policy_id(),
"runtime/y",
ts(1_000),
);
ok.jit_received(ts(1_001)).expect("a legal transition");
ok.started(1, ts(1_002)).expect("a legal transition");
ok.registered_idle(biggest, ts(1_003))
.expect("a legal transition");
store.record_attempt(&ok).expect("journalled");
assert_eq!(
store
.attempt(ok.id)
.expect("loads")
.expect("present")
.github_runner_id(),
Some(biggest)
);
let policy = ScalePolicy::new(
policy_id(),
ScaleTarget::repository("o/r").expect("valid"),
u64::MAX,
host_id(),
PolicyMode::monitor_only(),
CachePolicy::default(),
);
let error = store
.insert_policy(&policy)
.expect_err("an installation id SQLite cannot hold must not be truncated");
assert!(
matches!(
error,
StoreError::UnrepresentableInteger {
what: "policies.installation_id",
..
}
),
"got {error:?}"
);
}
#[test]
fn attempts_are_listed_oldest_first_and_can_be_filtered_by_policy() {
let store = store();
let other_policy = PolicyId::from_u128(0x0000_0011);
let first = RunnerAttempt::allocate(
AttemptId::from_u128(0xA1),
policy_id(),
"runtime/a1",
ts(1_000),
);
let second = RunnerAttempt::allocate(
AttemptId::from_u128(0xA2),
other_policy,
"runtime/a2",
ts(2_000),
);
let third = RunnerAttempt::allocate(
AttemptId::from_u128(0xA3),
policy_id(),
"runtime/a3",
ts(3_000),
);
for attempt in [&third, &first, &second] {
store.record_attempt(attempt).expect("journalled");
}
assert_eq!(
store
.attempts()
.expect("loads")
.iter()
.map(|a| a.created_at)
.collect::<Vec<_>>(),
vec![ts(1_000), ts(2_000), ts(3_000)],
"insertion order must not decide read order"
);
assert_eq!(
store.attempts_for_policy(policy_id()).expect("loads"),
vec![first.clone(), third]
);
assert!(store.remove_attempt(first.id).expect("removable"));
assert!(
!store.remove_attempt(first.id).expect("idempotent"),
"removing an absent attempt is not an error, it is a `false`"
);
assert_eq!(store.attempts().expect("loads").len(), 2);
}
#[test]
fn the_store_is_usable_as_a_shared_trait_object() {
let concrete = store();
RawHost::default().insert(&concrete);
let store: Arc<dyn Store> = Arc::new(concrete);
let handle = Arc::clone(&store);
let seen = std::thread::spawn(move || handle.host(host_id()).expect("loads").is_some())
.join()
.expect("the reader thread did not panic");
assert!(seen);
assert!(store.policies().expect("loads").is_empty());
}
#[test]
fn the_dump_names_every_table_and_reaches_every_column() {
let store = store();
let root = a_root("rman");
RawHost {
runner_root_override: Some(root.as_str().to_string()),
..RawHost::default()
}
.insert(&store);
RawPolicy {
workspace_mode: "persistent".to_string(),
workspace_path: Some(root.as_str().to_string()),
..RawPolicy::default()
}
.insert(&store);
RawAttempt {
workspace_mode: "persistent".to_string(),
workspace_slot: Some(1),
..RawAttempt::default()
}
.insert(&store);
let dump = store.dump_text().expect("dumpable");
for table in TABLES {
assert!(
dump.contains(&format!("-- table {table}")),
"{table} is missing from the dump"
);
}
assert!(dump.contains(&format!("-- schema version {SCHEMA_VERSION}")));
assert!(dump.contains("hosts.display_name=home-pc"));
assert!(dump.contains("policies.target_slug=o/r"));
assert!(dump.contains("attempts.outcome=NULL"));
assert!(
dump.contains("attempts.runtime_path=runtime/policy/attempt"),
"a dump that omitted a column would make the security scan vacuous"
);
for column in [
format!("hosts.runner_root_override={root}"),
format!("policies.workspace_path={root}"),
"policies.workspace_mode=persistent".to_string(),
"attempts.workspace_mode=persistent".to_string(),
"attempts.workspace_slot=1".to_string(),
] {
assert!(dump.contains(&column), "{column} is missing from the dump");
}
let reason = FailureReason::Other("no credential here".to_string());
assert!(json(&AttemptOutcome::failed(reason)).contains("no credential here"));
}
}