use areev_run_core::{EffectOutcome, FailCause, JournalKey, NodeExecutor};
use serde_json::Value;
use std::sync::mpsc;
use std::sync::Arc;
pub enum ExecResult {
Ok(Value),
Err { cause: FailCause, detail: String },
}
pub trait HostToolExecutor: Send + Sync {
fn execute(
&self,
tool_name: &str,
tool_hash: &str,
input: &Value,
idempotency_key: &str,
) -> ExecResult;
fn code_allowed(&self, _tool_hash: &str, _uri: &str) -> bool {
false
}
fn refusals(&self) -> Vec<crate::broker::EgressRefusal> {
Vec::new()
}
fn calls(&self) -> Vec<crate::broker::EgressCall> {
Vec::new()
}
fn blob_reads(&self) -> Vec<crate::broker::BlobRead> {
Vec::new()
}
fn bind_run_principal(&self, _principal: &str) {}
fn execute_code(
&self,
_tool_name: &str,
_tool_hash: &str,
code: &PreparedCode,
_input: &Value,
_idempotency_key: &str,
) -> ExecResult {
ExecResult::Err {
cause: FailCause::ExecutorError,
detail: format!(
"this host does not run code-carrying tools, so {} was not executed",
code.uri
),
}
}
fn runtime_supported(&self, _runtime: &str) -> bool {
false
}
}
#[derive(Default)]
pub struct ExecutorRegistry {
tools: std::collections::BTreeMap<String, Box<dyn HostToolExecutor>>,
}
impl ExecutorRegistry {
pub fn new() -> Self {
Self::default()
}
pub fn register(&mut self, tool_name: &str, exec: Box<dyn HostToolExecutor>) {
self.tools.insert(tool_name.to_string(), exec);
}
}
impl HostToolExecutor for ExecutorRegistry {
fn execute(
&self,
tool_name: &str,
tool_hash: &str,
input: &Value,
idempotency_key: &str,
) -> ExecResult {
match self.tools.get(tool_name) {
Some(t) => t.execute(tool_name, tool_hash, input, idempotency_key),
None => ExecResult::Err {
cause: FailCause::ExecutorError,
detail: format!("no executor registered for tool '{tool_name}'"),
},
}
}
}
#[derive(Clone)]
pub struct EgressHandle {
broker: Arc<crate::broker::Broker>,
}
impl EgressHandle {
pub fn new(broker: Arc<crate::broker::Broker>) -> Self {
EgressHandle { broker }
}
fn refusals(&self) -> Vec<crate::broker::EgressRefusal> {
self.broker.refusals()
}
fn calls(&self) -> Vec<crate::broker::EgressCall> {
self.broker.calls()
}
fn blob_reads(&self) -> Vec<crate::broker::BlobRead> {
self.broker.blob_reads()
}
fn token_for(&self, caller: &str) -> Option<&str> {
self.broker.token_for(caller)
}
fn declare(
&self,
caller: &str,
declaration: areev_core::types::capability::Declaration,
limits: crate::broker::CapabilityLimits,
) {
self.broker.declare(caller, declaration, limits)
}
fn bind_run_principal(&self, principal: &str) {
self.broker.bind_run_principal(principal)
}
fn env_for(&self, tool_name: &str) -> Vec<(&'static str, String)> {
match self.broker.token_for(tool_name) {
Some(tok) => vec![
("AREEV_EGRESS_URL", self.broker.url().to_string()),
("AREEV_EGRESS_TOKEN", tok.to_string()),
],
None => Vec::new(),
}
}
}
pub fn env_allow_policy(names: &str) -> (areev_core::proc::EnvPolicy, Vec<String>) {
let secrets = areev_core::proc::secret_env_vars();
let mut allow = areev_core::proc::EnvPolicy::minimal_allow();
let mut dropped = Vec::new();
for name in names.split(',').map(str::trim).filter(|s| !s.is_empty()) {
if secrets.iter().any(|s| s == name) {
dropped.push(name.to_string());
} else {
allow.push(name.to_string());
}
}
(areev_core::proc::EnvPolicy::ClearExcept { allow }, dropped)
}
pub struct CommandExecutor {
cmd: String,
timeout: Option<std::time::Duration>,
egress: Option<EgressHandle>,
env: areev_core::proc::EnvPolicy,
}
impl CommandExecutor {
pub fn new(cmd: &str) -> Self {
CommandExecutor {
cmd: cmd.to_string(),
timeout: Some(areev_core::proc::DEFAULT_TIMEOUT),
egress: None,
env: areev_core::proc::EnvPolicy::default(),
}
}
pub fn with_env_policy(mut self, env: areev_core::proc::EnvPolicy) -> Self {
self.env = env;
self
}
pub fn with_egress(mut self, egress: EgressHandle) -> Self {
self.egress = Some(egress);
self
}
pub fn with_timeout(mut self, timeout: Option<std::time::Duration>) -> Self {
self.timeout = timeout;
self
}
}
impl HostToolExecutor for CommandExecutor {
fn refusals(&self) -> Vec<crate::broker::EgressRefusal> {
self.egress.as_ref().map(|e| e.refusals()).unwrap_or_default()
}
fn calls(&self) -> Vec<crate::broker::EgressCall> {
self.egress.as_ref().map(|e| e.calls()).unwrap_or_default()
}
fn blob_reads(&self) -> Vec<crate::broker::BlobRead> {
self.egress.as_ref().map(|e| e.blob_reads()).unwrap_or_default()
}
fn bind_run_principal(&self, principal: &str) {
if let Some(e) = &self.egress {
e.bind_run_principal(principal);
}
}
fn execute(
&self,
tool_name: &str,
tool_hash: &str,
input: &Value,
idempotency_key: &str,
) -> ExecResult {
use areev_core::proc::{self, SpawnPolicy};
use std::process::Command;
#[cfg(not(windows))]
let mut shell = Command::new("/bin/sh");
#[cfg(not(windows))]
shell.arg("-c").arg(&self.cmd);
#[cfg(windows)]
let mut shell = Command::new("cmd");
#[cfg(windows)]
{
use std::os::windows::process::CommandExt;
shell.raw_arg("/C").raw_arg(&self.cmd);
}
let policy = SpawnPolicy {
timeout: self.timeout,
env: self.env.clone(),
..SpawnPolicy::default()
};
let mut env: Vec<(&str, &str)> = vec![
("AREEV_TOOL_NAME", tool_name),
("AREEV_TOOL_HASH", tool_hash),
("AREEV_IDEMPOTENCY_KEY", idempotency_key),
];
let brokered = self.egress.as_ref().map(|e| e.env_for(tool_name)).unwrap_or_default();
env.extend(brokered.iter().map(|(k, v)| (*k, v.as_str())));
let out = match proc::run(
shell,
Some(input.to_string().as_bytes()),
&env,
&policy,
) {
Ok(o) => o,
Err(e) => {
return ExecResult::Err {
cause: FailCause::ExecutorError,
detail: format!("spawn '{}': {e}", self.cmd),
}
}
};
if out.timed_out {
return ExecResult::Err {
cause: FailCause::Timeout,
detail: format!(
"tool command exceeded its {}s ceiling and was killed",
policy.timeout.map(|d| d.as_secs()).unwrap_or(0)
),
};
}
if let Some(why) = out.failure("tool command") {
return ExecResult::Err { cause: FailCause::ExecutorError, detail: why };
}
match serde_json::from_slice::<Value>(&out.stdout) {
Ok(v) => ExecResult::Ok(v),
Err(e) => ExecResult::Err {
cause: FailCause::ExecutorError,
detail: if out.stdout_truncated {
format!("tool output exceeded the capture cap and is not JSON: {e}")
} else {
format!("tool output is not JSON: {e}")
},
},
}
}
}
pub struct CodeExecutor {
inner: Arc<dyn HostToolExecutor>,
allowed: std::collections::BTreeSet<String>,
cache_dir: std::path::PathBuf,
timeout: Option<std::time::Duration>,
egress: Option<EgressHandle>,
sandbox_cmd: Option<Vec<String>>,
native_env: Option<areev_core::proc::EnvPolicy>,
}
impl CodeExecutor {
pub fn new(inner: Arc<dyn HostToolExecutor>) -> Self {
CodeExecutor {
inner,
allowed: Default::default(),
cache_dir: std::env::temp_dir().join("areev-executors"),
timeout: Some(areev_core::proc::DEFAULT_TIMEOUT),
egress: None,
sandbox_cmd: None,
native_env: None,
}
}
pub fn with_env_policy(mut self, env: areev_core::proc::EnvPolicy) -> Self {
self.native_env = Some(env);
self
}
pub fn sandbox_cmd(mut self, cmd: &str) -> Self {
let argv: Vec<String> = cmd.split_whitespace().map(str::to_string).collect();
if !argv.is_empty() {
self.sandbox_cmd = Some(argv);
}
self
}
pub fn with_egress(mut self, egress: EgressHandle) -> Self {
self.egress = Some(egress);
self
}
pub fn allow(mut self, addr: &str) -> Self {
self.allowed.insert(strip_cas(addr).to_ascii_lowercase());
self
}
pub fn cache_dir(mut self, dir: impl Into<std::path::PathBuf>) -> Self {
self.cache_dir = dir.into();
self
}
pub fn with_timeout(mut self, timeout: Option<std::time::Duration>) -> Self {
self.timeout = timeout;
self
}
fn materialize(&self, hex: &str, bytes: &[u8]) -> std::io::Result<std::path::PathBuf> {
let path = self.cache_dir.join(hex);
if path.exists() {
return Ok(path);
}
std::fs::create_dir_all(&self.cache_dir)?;
let tmp = self.cache_dir.join(format!("{hex}.{}.tmp", std::process::id()));
std::fs::write(&tmp, bytes)?;
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
std::fs::set_permissions(&tmp, std::fs::Permissions::from_mode(0o700))?;
}
std::fs::rename(&tmp, &path)?;
Ok(path)
}
fn register_capability(&self, tool_name: &str, code: &PreparedCode) -> Result<(), String> {
let Some(egress) = self.egress.as_ref() else {
return Err(format!(
"{} declares runtime \"wasm32-areev-io\" but this host configured no credential \
broker — a capability module needs --allow-host/--credential/--tool-egress",
code.uri
));
};
if egress.token_for(tool_name).is_none() {
return Err(format!(
"{} declares capabilities but the host granted tool '{tool_name}' no egress — \
add --tool-egress '{tool_name}:<credentials>:<methods>' (both may be empty for \
a module that only reads blobs)",
code.uri
));
}
let declared = match &code.capabilities {
Some(v) => areev_core::types::capability::Declaration::parse(v)
.map_err(|e| format!("{} declares malformed capabilities: {e}", code.uri))?,
None => areev_core::types::capability::Declaration::default(),
};
let mut limits = crate::broker::CapabilityLimits::default();
if let Some(l) = &code.limits {
if let Some(n) = l.get("max_calls").and_then(Value::as_u64) {
limits.max_calls = n.min(u64::from(u32::MAX)) as u32;
}
if let Some(n) = l.get("max_response_bytes").and_then(Value::as_u64) {
limits.max_response_bytes = usize::try_from(n).unwrap_or(usize::MAX);
}
}
egress.declare(tool_name, declared, limits);
Ok(())
}
}
pub(crate) fn strip_cas(uri: &str) -> &str {
uri.strip_prefix("cas://sha256:").unwrap_or(uri)
}
pub fn is_sandbox_runtime(runtime: &str) -> bool {
matches!(runtime, "wasm32-areev" | "wasm32-areev-io")
}
pub fn runtime_allows_capabilities(runtime: Option<&str>) -> bool {
runtime == Some("wasm32-areev-io")
}
impl HostToolExecutor for CodeExecutor {
fn execute(
&self,
tool_name: &str,
tool_hash: &str,
input: &Value,
idempotency_key: &str,
) -> ExecResult {
self.inner.execute(tool_name, tool_hash, input, idempotency_key)
}
fn calls(&self) -> Vec<crate::broker::EgressCall> {
match &self.egress {
Some(e) => e.calls(),
None => self.inner.calls(),
}
}
fn blob_reads(&self) -> Vec<crate::broker::BlobRead> {
match &self.egress {
Some(e) => e.blob_reads(),
None => self.inner.blob_reads(),
}
}
fn bind_run_principal(&self, principal: &str) {
match &self.egress {
Some(e) => e.bind_run_principal(principal),
None => self.inner.bind_run_principal(principal),
}
}
fn refusals(&self) -> Vec<crate::broker::EgressRefusal> {
match &self.egress {
Some(e) => e.refusals(),
None => self.inner.refusals(),
}
}
fn code_allowed(&self, _tool_hash: &str, uri: &str) -> bool {
self.allowed.contains(&strip_cas(uri).to_ascii_lowercase())
}
fn runtime_supported(&self, runtime: &str) -> bool {
is_sandbox_runtime(runtime) && self.sandbox_cmd.is_some()
}
fn execute_code(
&self,
tool_name: &str,
tool_hash: &str,
code: &PreparedCode,
input: &Value,
idempotency_key: &str,
) -> ExecResult {
let uri = code.uri.as_str();
if !self.code_allowed(tool_hash, uri) {
return ExecResult::Err {
cause: FailCause::ExecutorError,
detail: format!("{uri} is not pinned by this host"),
};
}
let hex = strip_cas(uri).to_ascii_lowercase();
let path = match self.materialize(&hex, &code.bytes) {
Ok(p) => p,
Err(e) => {
return ExecResult::Err {
cause: FailCause::ExecutorError,
detail: format!("could not materialize {uri}: {e}"),
}
}
};
use areev_core::proc::{self, EnvPolicy, SpawnPolicy};
let sandboxed = matches!(code.runtime.as_deref(), Some(rt) if rt != "native");
let mut spawned = path.display().to_string();
let cmd = match code.runtime.as_deref() {
None | Some("native") => std::process::Command::new(&path),
Some(rt) => {
let Some(argv) = self.sandbox_cmd.as_ref().filter(|_| is_sandbox_runtime(rt))
else {
return ExecResult::Err {
cause: FailCause::ExecutorError,
detail: format!(
"{uri} declares runtime {rt:?}, which this host cannot \
dispatch — configure --sandbox-cmd"
),
};
};
spawned.clone_from(&argv[0]);
let mut c = std::process::Command::new(&argv[0]);
c.args(&argv[1..]);
c.arg("--module").arg(&path);
if let Some(limits) = &code.limits {
if let Some(fuel) = limits.get("fuel").and_then(Value::as_u64) {
c.arg("--fuel").arg(fuel.to_string());
}
if let Some(pages) = limits.get("max_pages").and_then(Value::as_u64) {
c.arg("--max-pages").arg(pages.to_string());
}
if let Some(n) = limits.get("max_response_bytes").and_then(Value::as_u64) {
c.arg("--max-response-bytes").arg(n.to_string());
}
if let Some(n) = limits.get("max_blob_bytes").and_then(Value::as_u64) {
c.arg("--max-blob-bytes").arg(n.to_string());
}
}
if runtime_allows_capabilities(Some(rt)) {
if let Err(detail) = self.register_capability(tool_name, code) {
return ExecResult::Err {
cause: FailCause::ExecutorError,
detail,
};
}
c.arg("--allow-fetch");
if code
.capabilities
.as_ref()
.and_then(|v| areev_core::types::capability::Declaration::parse(v).ok())
.is_some_and(|d| d.declares_blob_read())
{
c.arg("--allow-blob");
}
}
c
}
};
let policy = SpawnPolicy {
timeout: self.timeout,
env: if sandboxed {
EnvPolicy::ClearExcept { allow: EnvPolicy::minimal_allow() }
} else {
self.native_env.clone().unwrap_or_default()
},
..SpawnPolicy::default()
};
let mut env: Vec<(&str, &str)> = vec![
("AREEV_TOOL_NAME", tool_name),
("AREEV_TOOL_HASH", tool_hash),
("AREEV_IDEMPOTENCY_KEY", idempotency_key),
("AREEV_EXECUTOR_URI", uri),
];
let brokered = self.egress.as_ref().map(|e| e.env_for(tool_name)).unwrap_or_default();
env.extend(brokered.iter().map(|(k, v)| (*k, v.as_str())));
let out = match proc::run(
cmd,
Some(input.to_string().as_bytes()),
&env,
&policy,
) {
Ok(o) => o,
Err(e) => {
return ExecResult::Err {
cause: FailCause::ExecutorError,
detail: format!("spawn {spawned}: {e}"),
}
}
};
if out.timed_out {
return ExecResult::Err {
cause: FailCause::Timeout,
detail: format!(
"executor {uri} exceeded its {}s ceiling and was killed",
policy.timeout.map(|d| d.as_secs()).unwrap_or(0)
),
};
}
if let Some(why) = out.failure("executor") {
return ExecResult::Err { cause: FailCause::ExecutorError, detail: why };
}
match serde_json::from_slice::<Value>(&out.stdout) {
Ok(v) => ExecResult::Ok(v),
Err(e) => ExecResult::Err {
cause: FailCause::ExecutorError,
detail: format!("executor output is not JSON: {e}"),
},
}
}
}
pub type TokenSink = Box<dyn FnMut(&str) + Send>;
pub struct PreparedLlm {
pub messages: Vec<areev_llm::ChatMessage>,
pub tools: Vec<areev_core::types::Tool>,
pub max_tokens: u32,
pub on_token: Option<TokenSink>,
}
pub struct DispatchJob {
pub key: JournalKey,
pub executor: NodeExecutor,
pub input: Value,
pub idempotency_key: String,
pub llm: Option<PreparedLlm>,
pub code: Option<PreparedCode>,
}
pub struct PreparedCode {
pub uri: String,
pub bytes: Vec<u8>,
pub runtime: Option<String>,
pub limits: Option<Value>,
pub capabilities: Option<Value>,
}
pub struct DispatchDone {
pub key: JournalKey,
pub executor: NodeExecutor,
pub outcome: EffectOutcome,
}
pub struct Pool {
tx: mpsc::Sender<DispatchJob>,
pub done_rx: mpsc::Receiver<DispatchDone>,
_workers: Vec<std::thread::JoinHandle<()>>,
}
impl Pool {
pub fn new(
executor: Arc<dyn HostToolExecutor>,
llm: Option<Arc<dyn areev_llm::ToolCallLlm>>,
workers: usize,
) -> Self {
let (tx, rx) = mpsc::channel::<DispatchJob>();
let rx = Arc::new(std::sync::Mutex::new(rx));
let (done_tx, done_rx) = mpsc::channel::<DispatchDone>();
let mut handles = Vec::new();
for _ in 0..workers.max(1) {
let rx = Arc::clone(&rx);
let done_tx = done_tx.clone();
let executor = Arc::clone(&executor);
let llm = llm.clone();
handles.push(std::thread::spawn(move || loop {
let job = {
let guard = rx.lock().expect("pool queue");
guard.recv()
};
let Ok(job) = job else { return };
if let Some(prepared) = job.llm {
let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(
|| run_llm_effect(llm.as_deref(), prepared),
))
.unwrap_or_else(|p| {
let msg = p
.downcast_ref::<&str>()
.map(|s| s.to_string())
.or_else(|| p.downcast_ref::<String>().cloned())
.unwrap_or_else(|| "llm executor panicked".into());
EffectOutcome::Failed {
cause: FailCause::ExecutorError,
detail: format!("llm panicked: {msg}"),
journal_bytes: 0,
}
});
let done = DispatchDone { key: job.key, executor: job.executor, outcome };
if done_tx.send(done).is_err() {
return;
}
continue;
}
let (tool_name, tool_hash) = match &job.executor {
NodeExecutor::Host { tool_name, tool_hash }
| NodeExecutor::Client { tool_name, tool_hash } => {
(tool_name.clone(), tool_hash.clone())
}
NodeExecutor::Subgraph { workflow_hash } => {
("mg:subgraph".to_string(), workflow_hash.clone())
}
NodeExecutor::Abstract { .. } => {
("mg:llm".to_string(), String::new())
}
};
let executed = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
match &job.code {
Some(c) => executor.execute_code(
&tool_name,
&tool_hash,
c,
&job.input,
&job.idempotency_key,
),
None => executor.execute(
&tool_name,
&tool_hash,
&job.input,
&job.idempotency_key,
),
}
}))
.unwrap_or_else(|p| {
let msg = p
.downcast_ref::<&str>()
.map(|s| s.to_string())
.or_else(|| p.downcast_ref::<String>().cloned())
.unwrap_or_else(|| "executor panicked".into());
ExecResult::Err {
cause: FailCause::ExecutorError,
detail: format!("executor panicked: {msg}"),
}
});
let outcome = match executed {
ExecResult::Ok(result) => {
let bytes = crate::journal::outcome_journal_bytes(
&EffectOutcome::Completed {
result: result.clone(),
journal_bytes: 0,
input_tokens: 0,
output_tokens: 0,
usd_micros: 0,
},
);
EffectOutcome::Completed {
result,
journal_bytes: bytes,
input_tokens: 0,
output_tokens: 0,
usd_micros: 0,
}
}
ExecResult::Err { cause, detail } => {
let bytes = detail.len() as u64;
EffectOutcome::Failed { cause, detail, journal_bytes: bytes }
}
};
let done = DispatchDone { key: job.key, executor: job.executor, outcome };
if done_tx.send(done).is_err() {
return;
}
}));
}
Pool { tx, done_rx, _workers: handles }
}
pub fn submit(&self, job: DispatchJob) {
let _ = self.tx.send(job);
}
}
fn run_llm_effect(
llm: Option<&dyn areev_llm::ToolCallLlm>,
prepared: crate::executor::PreparedLlm,
) -> EffectOutcome {
let Some(llm) = llm else {
return EffectOutcome::Failed {
cause: FailCause::Unknown,
detail: "no ToolCallLlm configured (RUN-E006 should have caught this at load)"
.into(),
journal_bytes: 0,
};
};
let request = areev_llm::ToolCallRequest {
system: Some(
"You are executing one node of a governed workflow. Use the \
offered tools when needed; when done, answer with the node's \
final result (a JSON object when structured output is wanted)."
.into(),
),
messages: prepared.messages,
tools: &prepared.tools,
tool_choice: areev_llm::ToolChoice::Auto,
max_tokens: prepared.max_tokens,
temperature: 0.0,
};
let result = match prepared.on_token {
Some(mut sink) => llm.call_streaming(&request, &mut *sink),
None => llm.call(&request),
};
match result {
Ok(resp) => {
let result = serde_json::json!({
"text": resp.text,
"tool_calls": resp.tool_calls.iter().map(|c| serde_json::json!({
"id": c.id,
"name": c.name,
"arguments": c.arguments_raw.clone()
.map(Value::String)
.unwrap_or_else(|| c.arguments.clone()),
})).collect::<Vec<_>>(),
"stop_reason": match resp.stop_reason {
areev_llm::StopReason::EndTurn => "end_turn",
areev_llm::StopReason::ToolUse => "tool_use",
areev_llm::StopReason::MaxTokens => "max_tokens",
areev_llm::StopReason::Other(_) => "other",
},
});
let bytes = crate::journal::outcome_journal_bytes(&EffectOutcome::Completed {
result: result.clone(),
journal_bytes: 0,
input_tokens: 0,
output_tokens: 0,
usd_micros: 0,
});
EffectOutcome::Completed {
result,
journal_bytes: bytes,
input_tokens: resp.usage.input_tokens,
output_tokens: resp.usage.output_tokens,
usd_micros: 0,
}
}
Err(e) => EffectOutcome::Failed {
cause: match (e.context_overflow, e.retryable) {
(true, _) => FailCause::ContextOverflow,
(false, true) => FailCause::ExecutorError,
(false, false) => FailCause::Unknown,
},
journal_bytes: e.message.len() as u64,
detail: e.message,
},
}
}