pub mod docker;
#[cfg(target_os = "macos")]
pub mod lume;
#[cfg(target_os = "linux")]
pub mod meda;
pub mod registry;
use async_trait::async_trait;
use std::time::Duration;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Deserialize, serde::Serialize)]
#[serde(rename_all = "lowercase")]
pub enum ExecutorKind {
Docker,
Meda,
Lume,
}
impl ExecutorKind {
pub const ALL: &'static [ExecutorKind] =
&[ExecutorKind::Docker, ExecutorKind::Meda, ExecutorKind::Lume];
pub fn name(self) -> &'static str {
match self {
ExecutorKind::Docker => "docker",
ExecutorKind::Meda => "meda",
ExecutorKind::Lume => "lume",
}
}
pub fn produced_os(self) -> &'static str {
match self {
ExecutorKind::Docker => "linux",
ExecutorKind::Meda => "linux",
ExecutorKind::Lume => "macos",
}
}
pub fn default_for_host_os(os: &str) -> Option<ExecutorKind> {
match os {
"linux" => Some(ExecutorKind::Meda),
"macos" => Some(ExecutorKind::Lume),
_ => None,
}
}
pub fn from_name(s: &str) -> Result<ExecutorKind, String> {
let lower = s.trim().to_ascii_lowercase();
ExecutorKind::ALL
.iter()
.find(|k| k.name() == lower)
.copied()
.ok_or_else(|| format!("unknown executor '{}'", s.trim()))
}
}
#[derive(Debug, Default, Clone)]
pub struct ExecutorFilter {
allow: Option<std::collections::HashSet<ExecutorKind>>,
}
impl ExecutorFilter {
pub fn allow_all() -> Self {
Self { allow: None }
}
pub fn allow_only(set: std::collections::HashSet<ExecutorKind>) -> Self {
Self { allow: Some(set) }
}
pub fn allows(&self, kind: ExecutorKind) -> bool {
self.allow.as_ref().is_none_or(|s| s.contains(&kind))
}
}
pub fn parse_executor_filter(raw: Option<&str>) -> Result<ExecutorFilter, String> {
let Some(s) = raw else {
return Ok(ExecutorFilter::allow_all());
};
let mut set = std::collections::HashSet::new();
for part in s.split(',') {
let trimmed = part.trim();
if trimmed.is_empty() {
continue;
}
set.insert(ExecutorKind::from_name(trimmed)?);
}
if set.is_empty() {
return Err("--executors cannot be empty".into());
}
Ok(ExecutorFilter::allow_only(set))
}
pub fn resolve_executor_kind(
top_executor: Option<&str>,
extra_config: Option<&serde_json::Value>,
os: &str,
) -> Result<ExecutorKind, String> {
if let Some(s) = top_executor {
if !s.is_empty() {
return ExecutorKind::from_name(s);
}
}
if let Some(cfg) = extra_config {
if let Some(name) = cfg.get("executor").and_then(|v| v.as_str()) {
return ExecutorKind::from_name(name);
}
if cfg.get("container").and_then(|v| v.as_bool()) == Some(true) {
return Ok(ExecutorKind::Docker);
}
}
ExecutorKind::default_for_host_os(os)
.ok_or_else(|| format!("no default executor for os '{os}'"))
}
pub fn executor_serves_os(kind: ExecutorKind, runner_os: &str) -> bool {
runner_os.eq_ignore_ascii_case(kind.produced_os())
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum GpuRequest {
None,
All,
Count(u32),
}
pub fn parse_gpu_request(value: Option<&serde_json::Value>) -> Result<GpuRequest, String> {
let v = match value {
None => return Ok(GpuRequest::None),
Some(v) if v.is_null() => return Ok(GpuRequest::None),
Some(v) => v,
};
if let Some(s) = v.as_str() {
let s = s.trim().to_ascii_lowercase();
if s.is_empty() || s == "none" {
return Ok(GpuRequest::None);
}
if s == "all" {
return Ok(GpuRequest::All);
}
if let Ok(n) = s.parse::<u32>() {
if n == 0 {
return Ok(GpuRequest::None);
}
return Ok(GpuRequest::Count(n));
}
return Err(format!("invalid gpu request: '{s}'"));
}
if let Some(n) = v.as_u64() {
return match u32::try_from(n) {
Ok(0) => Ok(GpuRequest::None),
Ok(n) => Ok(GpuRequest::Count(n)),
Err(_) => Err(format!("gpu count {n} out of range")),
};
}
Err(format!("gpu must be string or integer, got {v}"))
}
#[derive(Debug, Clone)]
pub struct RunnerLogin {
pub username: String,
#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
pub password: String,
}
#[derive(Debug, Clone)]
pub struct RunnerSpec {
pub name: String,
pub provision_script: String,
pub image: String,
pub cpu: u32,
pub memory_gb: u32,
#[cfg_attr(not(target_os = "linux"), allow(dead_code))]
pub disk_gb: u32,
pub gpu: GpuRequest,
pub docker_privileged: bool,
pub docker_mount_socket: bool,
pub login: RunnerLogin,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum RunnerState {
Healthy,
Starting,
Terminated {
exit_code: Option<i32>,
last_logs: String,
},
Absent,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct OwnedRunner {
pub name: String,
pub state: RunnerState,
}
#[derive(Debug, thiserror::Error)]
pub enum ProvisionError {
#[error("transient: {message}")]
Transient {
message: String,
diagnostics: serde_json::Map<String, serde_json::Value>,
},
#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
#[error("permanent: {0}")]
Permanent(String),
#[error("incompatible: {0}")]
Incompatible(String),
#[cfg_attr(not(target_os = "linux"), allow(dead_code))]
#[error("host_full ({code}): {message}")]
HostFull {
code: String,
message: String,
retry_after_secs: u64,
},
}
impl ProvisionError {
pub fn transient(msg: impl Into<String>) -> Self {
Self::Transient {
message: msg.into(),
diagnostics: serde_json::Map::new(),
}
}
#[cfg_attr(not(target_os = "linux"), allow(dead_code))]
pub fn transient_with(
msg: impl Into<String>,
diagnostics: serde_json::Map<String, serde_json::Value>,
) -> Self {
Self::Transient {
message: msg.into(),
diagnostics,
}
}
}
#[async_trait]
pub trait Executor: Send + Sync {
async fn inspect(&self, name: &str) -> Result<RunnerState, ProvisionError>;
async fn spawn(&self, spec: &RunnerSpec) -> Result<(), ProvisionError>;
async fn kill(&self, name: &str) -> Result<(), ProvisionError>;
async fn list_owned(&self) -> Result<Vec<OwnedRunner>, ProvisionError>;
fn settle_timeout(&self) -> Duration {
Duration::from_secs(120)
}
fn settle_poll_interval(&self) -> Duration {
Duration::from_secs(3)
}
fn validate(&self, _spec: &RunnerSpec) -> Result<(), ProvisionError> {
Ok(())
}
async fn prepare(&self, _spec: &RunnerSpec) -> Result<(), ProvisionError> {
Ok(())
}
async fn run_post_spawn(&self, _spec: &RunnerSpec) -> Result<(), ProvisionError> {
Ok(())
}
async fn provision(&self, spec: &RunnerSpec) -> Result<(), ProvisionError> {
self.validate(spec)?;
match self.inspect(&spec.name).await? {
RunnerState::Healthy | RunnerState::Starting => return Ok(()),
RunnerState::Terminated { .. } => {
self.kill(&spec.name).await?;
}
RunnerState::Absent => {}
}
self.prepare(spec).await?;
self.spawn(spec).await?;
let deadline = tokio::time::Instant::now() + self.settle_timeout();
loop {
tokio::time::sleep(self.settle_poll_interval()).await;
match self.inspect(&spec.name).await? {
RunnerState::Healthy => {
if let Err(e) = self.run_post_spawn(spec).await {
let _ = self.kill(&spec.name).await;
return Err(e);
}
return Ok(());
}
RunnerState::Starting => {
if tokio::time::Instant::now() >= deadline {
let _ = self.kill(&spec.name).await;
return Err(ProvisionError::transient(format!(
"runner '{}' did not reach Healthy within {:?}",
spec.name,
self.settle_timeout()
)));
}
}
RunnerState::Terminated {
exit_code,
last_logs,
} => {
let _ = self.kill(&spec.name).await;
return Err(ProvisionError::transient(format!(
"runner '{}' exited (code={:?}) during settle: {}",
spec.name, exit_code, last_logs
)));
}
RunnerState::Absent => {
return Err(ProvisionError::transient(format!(
"runner '{}' disappeared during settle",
spec.name
)));
}
}
}
}
async fn count_running(&self) -> Result<usize, ProvisionError> {
let runners = self.list_owned().await?;
Ok(runners
.into_iter()
.filter(|r| r.state == RunnerState::Healthy)
.count())
}
}
#[cfg(test)]
mod tests;