use regex::Regex;
#[derive(Debug, Clone, Default, PartialEq)]
pub enum JobStatus {
#[default]
Unknown,
Running,
Pending,
Completing,
Completed,
Timeout,
Cancelled,
Failed,
}
impl JobStatus {
pub fn abbrev(&self) -> &'static str {
match self {
JobStatus::Unknown => "?",
JobStatus::Running => "R",
JobStatus::Pending => "PD",
JobStatus::Completing => "CG",
JobStatus::Completed => "CD",
JobStatus::Timeout => "TO",
JobStatus::Cancelled => "CA",
JobStatus::Failed => "F",
}
}
pub fn priority(&self) -> usize {
match self {
JobStatus::Unknown => 0,
JobStatus::Pending => 1,
JobStatus::Running => 2,
JobStatus::Completing => 3,
JobStatus::Failed => 4,
JobStatus::Completed => 5,
JobStatus::Timeout => 6,
JobStatus::Cancelled => 7,
}
}
}
impl std::fmt::Display for JobStatus {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
let status = match self {
JobStatus::Unknown => "Unknown",
JobStatus::Running => "Running",
JobStatus::Pending => "Pending",
JobStatus::Completing => "Completing",
JobStatus::Completed => "Completed",
JobStatus::Timeout => "Timeout",
JobStatus::Cancelled => "Cancelled",
JobStatus::Failed => "Failed",
};
write!(f, "{}", status)
}
}
#[derive(Debug, Clone, Default, PartialEq)]
pub struct JobStats {
pub mem_efficiency: Option<f64>,
pub elapsed_frac_of_limit: Option<f64>,
}
impl JobStats {
pub fn has_any(&self) -> bool {
self.mem_efficiency.is_some() || self.elapsed_frac_of_limit.is_some()
}
}
#[derive(Debug, Clone, Default)]
pub struct Job {
pub id: String, pub name: String, pub status: JobStatus, pub time: String, pub partition: String, pub nodes: u32, pub workdir: String, pub command: String, pub output: Option<String>, pub reason: Option<String>,
pub priority: u64,
pub account: String,
pub qos: String,
pub cpus: u32,
pub nodelist: String,
pub stats: Option<Box<JobStats>>,
}
impl Job {
#[allow(clippy::too_many_arguments)]
pub fn new(
id: &str,
name: &str,
status: JobStatus,
time: &str,
partition: &str,
nodes: u32,
workdir: &str,
command: &str,
output: Option<String>,
) -> Self {
Self {
id: id.to_string(),
name: name.to_string(),
status,
time: time.to_string(),
partition: partition.to_string(),
nodes,
workdir: workdir.to_string(),
command: command.to_string(),
output,
reason: None,
priority: 0,
account: String::new(),
qos: String::new(),
cpus: 0,
nodelist: String::new(),
stats: None,
}
}
pub fn new_default() -> Self {
Self {
id: "123456".to_string(),
name: "jobname".to_string(),
status: JobStatus::Running,
time: "00:00:00".to_string(),
partition: "default".to_string(),
nodes: 1,
workdir: "/home/user".to_string(),
command: "/path/to/script".to_string(),
output: None,
reason: None,
priority: 0,
account: String::new(),
qos: String::new(),
cpus: 0,
nodelist: String::new(),
stats: None,
}
}
}
#[derive(Debug, Clone, PartialEq)]
pub enum ArrayTask {
Id(u64),
Range(String),
}
pub fn parse_array_id(id: &str) -> Option<(u64, ArrayTask)> {
let (base, task) = id.split_once('_')?;
let base: u64 = base.parse().ok()?;
if let Ok(task_id) = task.parse::<u64>() {
return Some((base, ArrayTask::Id(task_id)));
}
if task.starts_with('[') && task.ends_with(']') {
return Some((base, ArrayTask::Range(task.to_string())));
}
None
}
pub fn array_base_id(id: &str) -> Option<&str> {
parse_array_id(id)?;
id.split_once('_').map(|(base, _)| base)
}
pub fn pending_range_count(range: &str) -> u64 {
let inner = range.trim_start_matches('[').trim_end_matches(']');
let inner = inner.split('%').next().unwrap_or(inner);
inner
.split(',')
.map(|item| match item.trim().split_once('-') {
Some((start, end)) => match (start.parse::<u64>(), end.parse::<u64>()) {
(Ok(start), Ok(end)) if end >= start => end - start + 1,
_ => 1,
},
None => 1,
})
.sum::<u64>()
.max(1)
}
pub fn reason_code(raw: &str) -> &str {
let trimmed = raw.trim().trim_start_matches('(');
let end = trimmed
.find(|c: char| !c.is_ascii_alphanumeric() && c != '_')
.unwrap_or(trimmed.len());
&trimmed[..end]
}
pub fn explain_reason(code: &str) -> Option<&'static str> {
let explanation = match code {
"" | "None" => "waiting for the scheduler to evaluate the job",
"Priority" => "waiting: higher-priority jobs are ahead in the queue",
"Resources" => "waiting for the requested resources (nodes/CPUs/GPUs) to become free",
"Dependency" => "waiting for a job dependency to finish",
"DependencyNeverSatisfied" => {
"a dependency failed and can never be satisfied; the job will never start"
}
"QOSMaxCpuPerUserLimit" => {
"your per-user CPU limit of this QOS is reached; waits until your other jobs free CPUs"
}
"QOSMaxGRESPerUser" => "your per-user GPU/GRES limit of this QOS is reached",
"QOSGrpCpuLimit" => "the group CPU limit of this QOS is reached",
"QOSGrpGRES" => "the group GPU/GRES limit of this QOS is reached",
"AssocGrpCpuLimit" => "your association/account has reached its CPU limit",
"AssocGrpGRES" => "your association/account has reached its GPU/GRES limit",
"AssocGrpGRESMinutes" => "your association/account has used up its GPU/GRES-minutes budget",
"PartitionNodeLimit" => "the job requests more nodes than this partition allows",
"PartitionTimeLimit" => "the job requests more time than this partition allows",
"ReqNodeNotAvail" => {
"a requested node is not available (down, drained or reserved, e.g. for maintenance)"
}
"BeginTime" => "the requested start time of the job has not been reached yet",
"JobHeldUser" => "the job is held by the user; release it with 'scontrol release <jobid>'",
"JobHeldAdmin" => "the job is held by an administrator",
"NodeDown" => "a node required by the job is down",
"BadConstraints" => "the constraints of the job cannot be satisfied by any node",
"launch_failed_requeued_held" => {
"the job launch failed (often a node problem); it was requeued and held"
}
_ => return None,
};
Some(explanation)
}
impl Job {
pub fn get_jobname(&self) -> String {
self.name.clone()
}
pub fn get_stdout(&self) -> Option<String> {
match &self.output {
Some(output) => {
let mut output = output.to_string();
output = output.replace("%j", &self.id);
output = output.replace("%x", &self.name);
let re = Regex::new(r"%\d+j").unwrap();
Some(
re.replace_all(&output, |caps: ®ex::Captures| {
let num_str = &caps[0][1..caps[0].len() - 1];
if let Ok(num) = num_str.parse::<usize>() {
format!("{:0>width$}", self.id, width = num)
} else {
caps[0].to_string()
}
})
.to_string(),
)
}
None => None,
}
}
pub fn is_completed(&self) -> bool {
matches!(
self.status,
JobStatus::Completed | JobStatus::Failed | JobStatus::Timeout | JobStatus::Cancelled
)
}
}
#[cfg(test)]
mod tests {
use super::*;
fn job(id: &str, name: &str, output: Option<&str>) -> Job {
Job::new(
id,
name,
JobStatus::Running,
"00:00:00",
"compute",
1,
"/tmp",
"run.sh",
output.map(str::to_string),
)
}
#[test]
fn test_get_stdout_none_passthrough() {
assert_eq!(job("123", "myjob", None).get_stdout(), None);
}
#[test]
fn test_get_stdout_no_placeholders_unchanged() {
assert_eq!(
job("123", "myjob", Some("/logs/output.txt")).get_stdout(),
Some("/logs/output.txt".to_string())
);
}
#[test]
fn test_get_stdout_job_id_expansion() {
assert_eq!(
job("123", "myjob", Some("slurm-%j.out")).get_stdout(),
Some("slurm-123.out".to_string())
);
}
#[test]
fn test_get_stdout_job_name_expansion() {
assert_eq!(
job("123", "myjob", Some("%x.log")).get_stdout(),
Some("myjob.log".to_string())
);
}
#[test]
fn test_get_stdout_combined_expansion() {
assert_eq!(
job("123", "myjob", Some("/logs/%x_%j.out")).get_stdout(),
Some("/logs/myjob_123.out".to_string())
);
}
#[test]
fn test_get_stdout_zero_padded_id() {
assert_eq!(
job("123", "myjob", Some("slurm-%8j.out")).get_stdout(),
Some("slurm-00000123.out".to_string())
);
}
#[test]
fn test_get_stdout_padding_width_smaller_than_id() {
assert_eq!(
job("123456", "myjob", Some("%2j.out")).get_stdout(),
Some("123456.out".to_string())
);
}
#[test]
fn test_reason_code_extraction() {
assert_eq!(reason_code("Priority"), "Priority");
assert_eq!(reason_code("(Priority)"), "Priority");
assert_eq!(reason_code(" Resources "), "Resources");
assert_eq!(
reason_code("ReqNodeNotAvail, UnavailableNodes:n[1-2]"),
"ReqNodeNotAvail"
);
assert_eq!(
reason_code("launch_failed_requeued_held"),
"launch_failed_requeued_held"
);
assert_eq!(reason_code(""), "");
}
#[test]
fn test_explain_reason_known_codes() {
for code in [
"Priority",
"Resources",
"Dependency",
"DependencyNeverSatisfied",
"QOSMaxCpuPerUserLimit",
"QOSMaxGRESPerUser",
"QOSGrpCpuLimit",
"QOSGrpGRES",
"AssocGrpCpuLimit",
"AssocGrpGRES",
"AssocGrpGRESMinutes",
"PartitionNodeLimit",
"PartitionTimeLimit",
"ReqNodeNotAvail",
"BeginTime",
"JobHeldUser",
"JobHeldAdmin",
"NodeDown",
"BadConstraints",
"launch_failed_requeued_held",
"None",
"",
] {
assert!(
explain_reason(code).is_some(),
"no explanation for {:?}",
code
);
}
assert_eq!(
explain_reason("Priority"),
Some("waiting: higher-priority jobs are ahead in the queue")
);
}
#[test]
fn test_explain_reason_unknown_code_is_none() {
assert_eq!(explain_reason("SomeNewSlurmReason"), None);
assert_eq!(explain_reason("priority"), None); }
#[test]
fn test_job_stats_has_any() {
assert!(!JobStats::default().has_any());
assert!(JobStats {
mem_efficiency: Some(0.5),
..JobStats::default()
}
.has_any());
assert!(JobStats {
elapsed_frac_of_limit: Some(0.1),
..JobStats::default()
}
.has_any());
}
#[test]
fn test_parse_array_id_task() {
assert_eq!(parse_array_id("12345_7"), Some((12345, ArrayTask::Id(7))));
assert_eq!(array_base_id("12345_7"), Some("12345"));
}
#[test]
fn test_parse_array_id_pending_range() {
assert_eq!(
parse_array_id("12345_[8-99]"),
Some((12345, ArrayTask::Range("[8-99]".to_string())))
);
assert_eq!(array_base_id("12345_[8-99]"), Some("12345"));
}
#[test]
fn test_parse_array_id_non_array_ids() {
assert_eq!(parse_array_id("12345"), None);
assert_eq!(array_base_id("12345"), None);
assert_eq!(parse_array_id("abc_1"), None);
assert_eq!(parse_array_id("12345_abc"), None);
assert_eq!(parse_array_id(""), None);
}
#[test]
fn test_pending_range_count() {
assert_eq!(pending_range_count("[8-99]"), 92);
assert_eq!(pending_range_count("[0]"), 1);
assert_eq!(pending_range_count("[1,3,5]"), 3);
assert_eq!(pending_range_count("[1-3,7,10-11]"), 6);
assert_eq!(pending_range_count("[0-15%4]"), 16);
assert_eq!(pending_range_count("[garbage]"), 1);
assert_eq!(pending_range_count(""), 1);
}
#[test]
fn test_status_abbrev() {
assert_eq!(JobStatus::Running.abbrev(), "R");
assert_eq!(JobStatus::Pending.abbrev(), "PD");
assert_eq!(JobStatus::Completing.abbrev(), "CG");
assert_eq!(JobStatus::Completed.abbrev(), "CD");
assert_eq!(JobStatus::Failed.abbrev(), "F");
assert_eq!(JobStatus::Timeout.abbrev(), "TO");
assert_eq!(JobStatus::Cancelled.abbrev(), "CA");
assert_eq!(JobStatus::Unknown.abbrev(), "?");
}
#[test]
fn test_get_stdout_array_job_id() {
assert_eq!(
job("123_4", "myjob", Some("%j.out")).get_stdout(),
Some("123_4.out".to_string())
);
assert_eq!(
job("123_4", "myjob", Some("%8j.out")).get_stdout(),
Some("000123_4.out".to_string())
);
}
}