use std::collections::{BTreeMap, BTreeSet};
use std::path::{Component, Path, PathBuf};
use serde_json::Value;
use argonauts_core::TaskId;
use argonauts_core::events::RunEventKind;
use argonauts_core::events::run::RunMode;
use argonauts_core::events::workflow::{
PhaseMark, StepEnd, StepFinished, StepStarted, WorkflowDeclared, WorkflowEvent, WorkflowStarted,
};
use argonauts_core::future::BoxFuture;
use argonauts_core::names::{ModelRef, WorkflowId};
use argonauts_core::paths::Paths;
use argonauts_core::settings::Settings;
use argonauts_core::settings::protocol::UnderstandWhen;
use argonauts_core::workflow::{
StandardPhase, Step, StepRef, Workflow, WorkflowParams, WorkflowRef,
};
use argonauts_core::{canonical_json, sha256_hex, workflow_files};
use crate::engine::host::Host;
use crate::error::{EngineError, EngineResult};
use crate::goal::pursue;
use crate::roles;
use crate::state::RunState;
use crate::state::ids::IdKind;
use crate::task;
use crate::workflow::validate::{self, Context};
use crate::workflow::{
Flow, StepAt, StepCall, StepDone, builtin, change, conditions, describe, kinds, params, replay,
shape, yaml,
};
use argonauts_core::steps::StepName;
pub fn workflow_for_run(host: &Host) -> EngineResult<(Option<WorkflowRef>, Workflow)> {
let running = host.read(|state| {
let root = state.workflows.root.as_ref()?;
let record = state.workflows.by_id.get(root)?;
Some((record.from.clone(), record.workflow.clone()))
});
if let Some(running) = running {
return Ok(running);
}
let in_use = phases_in_use(host);
let configured = lookup(host, in_use.as_str());
if host.read(run_mode) != Some(RunMode::Captain) {
return Ok((Some(in_use), configured?));
}
if let Ok(workflow) = configured
&& builds_goal(&workflow)
{
return Ok((Some(in_use), workflow));
}
let name = WorkflowRef::new(captain_workflow(host.settings()));
let workflow = lookup(host, name.as_str())?;
Ok((Some(name), workflow))
}
pub fn run_mode(state: &RunState) -> Option<RunMode> {
state
.run
.mode
.or_else(|| state.run.started.as_ref().map(|started| started.mode))
}
pub fn crew_run(state: &RunState, id: &WorkflowId) -> bool {
run_mode(state) != Some(RunMode::Captain) && !in_captain_workflow(state, id)
}
pub fn in_captain_workflow(state: &RunState, id: &WorkflowId) -> bool {
let mut at = state.workflows.by_id.get(id);
while let Some(record) = at {
if builds_goal(&record.workflow) {
return true;
}
at = record
.parent
.as_ref()
.and_then(|parent| state.workflows.by_id.get(&parent.workflow));
}
false
}
pub fn captain_workflow(settings: &Settings) -> &'static str {
if settings.captain.tries > 1 {
"best-of-n"
} else if settings.captain.review {
"maker-checker"
} else {
"captain"
}
}
pub fn crew_workflow(host: &Host) -> WorkflowRef {
let configured = phases_in_use(host);
match lookup(host, configured.as_str()) {
Ok(workflow) if builds_goal(&workflow) => WorkflowRef::new(CAPTAIN_CREW_WORKFLOW),
_ => configured,
}
}
pub fn phases_in_use(host: &Host) -> WorkflowRef {
let name = host.settings().workflow.as_str();
if has_phases(host, name) {
WorkflowRef::new(name)
} else {
WorkflowRef::new(BASE_PHASES_WORKFLOW)
}
}
fn has_phases(host: &Host, name: &str) -> bool {
if Path::new(name).is_absolute() || builtin::source(name).is_some() {
return true;
}
let stem = name.strip_suffix(".star").unwrap_or(name);
let waiting = crate::prompts::files::waiting(host);
workflow_dirs(host).iter().any(|dir| {
let there = |file: String| {
let place = dir.join(file);
(place.is_file() && !waiting.contains(&place)).then_some(place)
};
let own_file = there(format!("{stem}.toml"))
.and_then(|place| std::fs::read_to_string(place).ok())
.is_some_and(|text| workflow_files::has_table(&text, workflow_files::PHASES_TABLE));
own_file
|| there(format!("{stem}.yaml")).is_some()
|| there(format!("{stem}.star")).is_some()
})
}
pub const CAPTAIN_CREW_WORKFLOW: &str = "orchestrator";
pub const BASE_PHASES_WORKFLOW: &str = "council";
pub fn builds_goal(workflow: &Workflow) -> bool {
fn any(steps: &[Step]) -> bool {
steps
.iter()
.any(|step| task::builds_goal(step) || any(&step.steps))
}
workflow.phases.iter().any(|phase| any(&phase.steps))
}
pub fn settings_params(workflow: &Workflow, settings: &Settings) -> WorkflowParams {
let mut values = WorkflowParams::default();
if settings.captain.tries > 1 && workflow.params.0.contains_key(TRIES) {
values
.0
.insert(TRIES.to_owned(), Value::from(settings.captain.tries));
}
values
}
pub const TRIES: &str = "tries";
pub fn workflow_dirs(host: &Host) -> Vec<PathBuf> {
let mut dirs = vec![host.repo().root.join(".argonauts/workflows")];
if let Ok(paths) = Paths::from_env() {
dirs.push(paths.home().join("workflows"));
}
dirs
}
pub fn lookup(host: &Host, name: &str) -> EngineResult<Workflow> {
if Path::new(name).is_absolute() {
return workflow_beside_learned(host, Path::new(name));
}
let plain = Path::new(name)
.components()
.all(|component| matches!(component, Component::Normal(_)));
if name.is_empty() || !plain || name.contains('/') {
return Err(EngineError::Refused(format!(
"`{name}` isn't a workflow name"
)));
}
let stem = name.strip_suffix(".star").unwrap_or(name);
let own_file = format!("{stem}.toml");
let file = format!("{stem}.yaml");
let star = format!("{stem}.star");
let waiting = crate::prompts::files::waiting(host);
let read = |place: &Path| match std::fs::read_to_string(place) {
Ok(text) => Ok(Some(text)),
Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(None),
Err(err) => Err(EngineError::from(err)),
};
for dir in workflow_dirs(host) {
let refused =
|place: &Path, err: String| EngineError::Refused(format!("{}: {err}", place.display()));
let place = dir.join(&own_file);
if stem == name
&& !waiting.contains(&place)
&& let Some(text) = read(&place)?
&& let Some(workflow) =
workflow_files::phases(&text, stem).map_err(|err| refused(&place, err))?
{
return Ok(workflow);
}
let place = dir.join(&file);
if stem == name
&& !waiting.contains(&place)
&& let Some(text) = read(&place)?
{
return yaml::parse(&text).map_err(|err| refused(&place, err));
}
let place = dir.join(&star);
if place.is_file() && !waiting.contains(&place) {
return argonauts_learning::learned_workflows::loops::build_file(&place)
.map_err(|err| EngineError::Refused(format!("{}: {err}", place.display())));
}
}
if builtin::source(name).is_some() {
return builtin::load(name).map_err(EngineError::Refused);
}
Err(EngineError::Refused(format!(
"no workflow is named `{name}`: neither .argonauts/workflows/ nor ~/.argonauts/workflows/ has {own_file} with phases or {file}, and it isn't built in"
)))
}
fn workflow_beside_learned(host: &Host, path: &Path) -> EngineResult<Workflow> {
let refused = |why: String| EngineError::Refused(format!("{}: {why}", path.display()));
let Some(file) =
crate::run_review::learned_workflow::learned_workflow_file(&host.repo().run_dir, path)
else {
return Err(refused(
"a phases file named by its full path must be a file beside this run's learned workflow".to_owned(),
));
};
if file
.extension()
.is_some_and(|extension| extension == "star")
{
return argonauts_learning::learned_workflows::loops::build_file(&file)
.map_err(|err| refused(err.to_string()));
}
yaml::parse(&std::fs::read_to_string(&file)?).map_err(|err| refused(err.to_string()))
}
pub fn allowed_workers(settings: &Settings) -> Vec<ModelRef> {
let mut allowed = vec![settings.crew.main.clone()];
allowed.extend(settings.crew.members());
for role in settings.roles.values() {
allowed.extend(role.models.iter().cloned());
}
allowed.sort();
allowed.dedup();
allowed
}
pub fn version(workflow: &Workflow) -> EngineResult<String> {
let value = serde_json::to_value(workflow)?;
Ok(sha256_hex(canonical_json(&value).as_bytes()))
}
pub fn check(
host: &Host,
workflow: &Workflow,
depth: u32,
) -> Result<(), Vec<validate::WorkflowError>> {
let settings = host.settings();
let allowed = allowed_workers(settings);
let limit_left_usd = host.read(|state| {
let spend = state.run.spend.as_ref()?;
let limit = spend.limit.as_ref()?;
(limit.unit == argonauts_core::settings::limit::LimitUnit::Usd)
.then_some(limit.amount - spend.spent.cost_usd)
});
let providers: BTreeMap<ModelRef, String> = allowed
.iter()
.map(|worker| (worker.clone(), roles::provider(host, worker)))
.collect();
let lookup = |name: &str| lookup(host, name).map_err(|err| err.to_string());
let as_you_asked = host.read(|state| {
let instructions = state
.run
.started
.as_ref()
.map(|started| started.instructions.as_slice())
.unwrap_or_default();
validate::roles_as_you_asked(instructions, settings)
});
validate::validate(
workflow,
&Context {
settings,
allowed: &allowed,
providers: &providers,
lookup: &lookup,
depth,
limit_left_usd,
as_you_asked: &as_you_asked,
},
)
}
pub fn start(
host: &Host,
from: Option<WorkflowRef>,
workflow: Workflow,
parent: Option<StepRef>,
) -> EngineResult<WorkflowId> {
start_as(host, None, from, workflow, parent)
}
pub fn start_as(
host: &Host,
id: Option<WorkflowId>,
from: Option<WorkflowRef>,
workflow: Workflow,
parent: Option<StepRef>,
) -> EngineResult<WorkflowId> {
let workflow = match parent {
None => params::fill(&workflow, &settings_params(&workflow, host.settings())).map_err(
|errors| {
EngineError::Refused(format!(
"workflow `{}`: {}",
workflow.name,
errors.join("; ")
))
},
)?,
Some(_) => workflow,
};
let version = version(&workflow)?;
let existing = host.read(|state| {
if parent.is_some() {
return None;
}
let workflows = &state.workflows;
let candidate = workflows.by_id.get(workflows.root.as_ref()?)?;
(!candidate.finished && candidate.from == from && candidate.version == version)
.then(|| candidate.id.clone())
});
if let Some(id) = existing {
return Ok(id);
}
let depth = match &parent {
None => 0,
Some(step) => host.read(|state| state.workflows.depth(&step.workflow)) + 1,
};
check(host, &workflow, depth).map_err(|errors| {
EngineError::Refused(format!(
"workflow `{}` isn't valid: {}",
workflow.name,
errors
.iter()
.map(ToString::to_string)
.collect::<Vec<_>>()
.join("; ")
))
})?;
let id = match id {
Some(id) => id,
None => WorkflowId::numbered(host.allocate(IdKind::Workflow)?),
};
host.record(
None,
None,
RunEventKind::Workflow(WorkflowEvent::Started(Box::new(WorkflowStarted {
id: id.clone(),
from,
version,
parent,
workflow,
}))),
)?;
Ok(id)
}
pub fn run<'a>(host: &'a Host, id: &'a WorkflowId) -> BoxFuture<'a, EngineResult<Flow>> {
Box::pin(async move {
let recorded = host.read(|state| {
let record = state.workflows.by_id.get(id)?;
Some(match replay::failed(record) {
Some(progress) => match replay::step(state, host, record, progress) {
Flow::Continue => Err(failed(&progress.step, progress.summary.as_deref())),
flow => Ok(flow),
},
None => Ok(replay::workflow(state, host, record)),
})
});
match recorded {
Some(Err(err)) => return Err(err),
Some(Ok(flow)) if flow != Flow::Continue => return Ok(flow),
_ => {}
}
let depth = host.read(|state| state.workflows.depth(id));
let adapts = host.settings().workflows.adapt_at_phase_ends
&& host.read(|state| crew_run(state, id) && !crate::script::runs(state, id));
let mut adapted = BTreeSet::new();
loop {
declare(host, id)?;
let Some((workflow, next, started)) = host.read(|state| {
let record = state.workflows.by_id.get(id)?;
let next = record.next_phase();
Some((record.workflow.clone(), next, record.phase_started(next)))
}) else {
return Err(EngineError::Refused(format!(
"no workflow {id} was started"
)));
};
let Some(phase) = workflow.phases.get(next as usize) else {
return Ok(Flow::Continue);
};
let mark = PhaseMark {
id: id.clone(),
index: next,
name: phase.name.clone(),
};
if next > 0
&& !started
&& adapts
&& !host.read(pursue::plans_its_rounds)
&& adapted.insert(next)
{
change::adapt_at_phase_end(host, id, next - 1).await?;
continue;
}
if pursue::begin_try(host, id, next)? {
continue;
}
for (index, step) in phase.steps.iter().enumerate() {
let call = StepCall {
at: StepAt {
step: StepRef {
workflow: id.clone(),
phase: next,
path: vec![index as u32],
},
round: 1,
},
depth,
standard: phase.standard,
step: step.clone(),
item: None,
};
let done = run_step(host, &call).await?;
if done.flow != Flow::Continue {
end_phase(host, &mark, done.end)?;
skip_the_rest(host, id, next + 1, &done.flow)?;
return Ok(done.flow);
}
if done.end == StepEnd::Failed {
end_phase(host, &mark, StepEnd::Failed)?;
return Err(failed(&call.at.step, done.summary.as_deref()));
}
}
end_phase(host, &mark, StepEnd::Done)?;
}
})
}
fn declare(host: &Host, id: &WorkflowId) -> EngineResult<()> {
let Some(mut declared) = host.read(|state| {
let record = state.workflows.by_id.get(id)?;
(record.declared.as_ref() != Some(&record.version))
.then(|| WorkflowDeclared::of(id, &record.workflow))
}) else {
return Ok(());
};
if host.read(|state| crate::script::runs(state, id)) {
declared.required = crate::script::required::names(host);
}
host.record(
None,
None,
RunEventKind::Workflow(WorkflowEvent::Declared(Box::new(declared))),
)?;
Ok(())
}
fn start_phase(host: &Host, step: &StepRef) -> EngineResult<()> {
if step.path.len() != 1 {
return Ok(());
}
let Some(mark) = host.read(|state| {
let record = state.workflows.by_id.get(&step.workflow)?;
let phase = record.workflow.phases.get(step.phase as usize)?;
(!record.phases_started.contains(&step.phase)).then(|| PhaseMark {
id: step.workflow.clone(),
index: step.phase,
name: phase.name.clone(),
})
}) else {
return Ok(());
};
host.record(
None,
None,
RunEventKind::Workflow(WorkflowEvent::PhaseStarted(mark)),
)?;
Ok(())
}
fn review_phase(state: &RunState, step: &StepRef) -> Option<PhaseMark> {
let record = state.workflows.by_id.get(&step.workflow)?;
let target = record.step_definition(step)?.review_in.as_deref()?;
let index = record
.workflow
.phases
.iter()
.position(|phase| phase.name == target && phase.steps.is_empty())?;
let index = u32::try_from(index)
.ok()
.filter(|index| *index > step.phase)?;
Some(PhaseMark {
id: step.workflow.clone(),
index,
name: target.to_owned(),
})
}
pub fn task_step(state: &RunState, task: &TaskId) -> Option<StepRef> {
state.sessions.in_order().into_iter().find_map(|session| {
(session.task.as_ref() == Some(task) && session.started.step == Some(StepName::Build))
.then(|| session.started.workflow_step.clone())
.flatten()
})
}
pub fn review_at(state: &RunState, task: &TaskId) -> Option<StepAt> {
let step = task_step(state, task)?;
if let Some(mark) = review_phase(state, &step) {
return Some(StepAt {
step: StepRef {
workflow: mark.id,
phase: mark.index,
path: Vec::new(),
},
round: 1,
});
}
let round = state
.workflows
.by_id
.get(&step.workflow)?
.steps
.iter()
.filter(|progress| progress.step == step)
.map(|progress| progress.round)
.max()
.unwrap_or(1);
Some(StepAt { step, round })
}
pub fn review_started(host: &Host, step: &StepRef) -> EngineResult<()> {
let Some(mark) = host.read(|state| {
review_phase(state, step).filter(|mark| {
state
.workflows
.by_id
.get(&mark.id)
.is_some_and(|record| !record.phases_started.contains(&mark.index))
})
}) else {
return Ok(());
};
host.record(
None,
None,
RunEventKind::Workflow(WorkflowEvent::PhaseStarted(mark)),
)?;
Ok(())
}
fn end_review_phase(host: &Host, step: &StepRef) -> EngineResult<()> {
let Some(mark) = host.read(|state| {
review_phase(state, step).filter(|mark| {
state.workflows.by_id.get(&mark.id).is_some_and(|record| {
record.phases_started.contains(&mark.index)
&& !record.phases_ended.contains(&mark.index)
})
})
}) else {
return Ok(());
};
host.record(
None,
None,
RunEventKind::Workflow(WorkflowEvent::PhaseEnded {
phase: mark,
end: StepEnd::Done,
}),
)?;
Ok(())
}
fn end_phase(host: &Host, mark: &PhaseMark, end: StepEnd) -> EngineResult<()> {
let Some(skipped) = host.read(|state| {
let record = state.workflows.by_id.get(&mark.id)?;
if record.phases_ended.contains(&mark.index) || record.phases_skipped.contains(&mark.index)
{
return None;
}
let tops: Vec<_> = record
.steps
.iter()
.filter(|progress| progress.step.phase == mark.index && progress.step.path.len() == 1)
.collect();
let all_skipped = !tops.is_empty()
&& tops
.iter()
.all(|progress| progress.end == Some(StepEnd::Skipped));
Some(all_skipped.then(|| {
tops.iter()
.find_map(|progress| progress.summary.clone())
.unwrap_or_else(|| "nothing in it had to run".to_owned())
}))
}) else {
return Ok(());
};
let event = match skipped {
Some(why) if end != StepEnd::Failed => WorkflowEvent::PhaseSkipped {
phase: mark.clone(),
why,
},
_ => WorkflowEvent::PhaseEnded {
phase: mark.clone(),
end,
},
};
host.record(None, None, RunEventKind::Workflow(event))?;
Ok(())
}
fn skip_the_rest(host: &Host, id: &WorkflowId, from: u32, flow: &Flow) -> EngineResult<()> {
let why = match flow {
Flow::Switch(to) => format!("the run switched to {to}"),
Flow::Finish(_) | Flow::Continue => "the run ended before it".to_owned(),
};
let marks = host.read(|state| {
let Some(record) = state.workflows.by_id.get(id) else {
return Vec::new();
};
record
.workflow
.phases
.iter()
.enumerate()
.skip(from as usize)
.filter_map(|(index, phase)| {
let index = u32::try_from(index).ok()?;
let recorded = record.phases_started.contains(&index)
|| record.phases_ended.contains(&index)
|| record.phases_skipped.contains(&index);
(!recorded).then(|| PhaseMark {
id: id.clone(),
index,
name: phase.name.clone(),
})
})
.collect::<Vec<_>>()
});
for mark in marks {
host.record(
None,
None,
RunEventKind::Workflow(WorkflowEvent::PhaseSkipped {
phase: mark,
why: why.clone(),
}),
)?;
}
Ok(())
}
fn failed(step: &StepRef, summary: Option<&str>) -> EngineError {
EngineError::Refused(format!(
"{} failed: {}",
describe(step),
summary.unwrap_or("it gave no reason")
))
}
pub fn run_step<'a>(host: &'a Host, call: &'a StepCall) -> BoxFuture<'a, EngineResult<StepDone>> {
Box::pin(async move {
let recorded = host.read(|state| {
let record = state.workflows.by_id.get(call.workflow())?;
let progress = record.round(&call.at.step, call.at.round)?;
Some(progress.end.map(|end| StepDone {
end,
summary: progress.summary.clone(),
flow: replay::step(state, host, record, progress),
}))
});
match recorded {
Some(Some(done)) => return Ok(done),
Some(None) => {}
None => {
if let Some(done) = understand_off(host, call).await? {
record_finished(host, call, &done)?;
return Ok(done);
}
let holds = match &call.step.when {
Some(condition) => host
.read(|state| {
conditions::holds(state, host.settings(), condition, &call.at)
})
.map_err(EngineError::Refused)?,
None => true,
};
if !holds {
let condition = call.step.when.as_deref().unwrap_or_default();
let done = StepDone::skipped(format!("{condition} doesn't hold"));
record_finished(host, call, &done)?;
return Ok(done);
}
start_phase(host, &call.at.step)?;
record_started(host, call)?;
}
}
let done = kinds::execute(host, call).await?;
record_finished(host, call, &done)?;
end_review_phase(host, &call.at.step)?;
Ok(done)
})
}
async fn understand_off(host: &Host, call: &StepCall) -> EngineResult<Option<StepDone>> {
let off = host.settings().understand.when == UnderstandWhen::Off
&& call.standard == StandardPhase::Understand
&& call.at.step.path.len() == 1
&& host.read(run_mode) == Some(RunMode::Crew);
if !off {
return Ok(None);
}
let asked_later = host.read(|state| {
state
.workflows
.by_id
.get(&call.at.step.workflow)
.is_some_and(|record| {
shape::understand_approval_runs(&record.workflow, UnderstandWhen::Off)
})
});
if !asked_later {
crate::captain::prompt_is_the_goal(host).await?;
}
Ok(Some(StepDone::skipped(
"understand.when is off, so your prompt is the goal",
)))
}
fn record_started(host: &Host, call: &StepCall) -> EngineResult<()> {
host.record(
None,
None,
RunEventKind::Workflow(WorkflowEvent::StepStarted(StepStarted {
step: call.at.step.clone(),
kind: call.step.kind,
standard: call.standard,
round: call.at.round,
})),
)?;
Ok(())
}
fn record_finished(host: &Host, call: &StepCall, done: &StepDone) -> EngineResult<()> {
host.record(
None,
None,
RunEventKind::Workflow(WorkflowEvent::StepFinished(StepFinished {
step: call.at.step.clone(),
kind: call.step.kind,
standard: call.standard,
round: call.at.round,
end: done.end,
summary: done.summary.clone(),
})),
)?;
Ok(())
}