use std::collections::HashMap;
use std::sync::Arc;
use areev_cal::AreevFacade;
use areev_core::error::{AreevError, Hash};
use areev_core::types::{ContentRef, Event, Grain, GrainType, Observation, Trigger, TriggerKind};
use areev_run::HostToolExecutor;
use areev_store::TriggerState;
use serde::Serialize;
use sha2::{Digest, Sha256};
use crate::clock::Clock;
use crate::connector::{
dedup_value, rewrite_blob_refs, PollItem, PollRequest, PollResponse,
MAX_BLOB_BYTES_PER_ITEM, MAX_BLOB_BYTES_PER_RESPONSE,
};
use crate::error::{Result, TriggerError};
use crate::schedule;
pub const TRIGGER_NS: &str = "agent:triggers";
const MAX_TRIGGERS_SCANNED: usize = 4096;
#[derive(Debug, Clone)]
pub struct EvalOptions {
pub only: Option<String>,
pub lease: std::time::Duration,
pub max_items: usize,
pub dry_run: bool,
pub node: String,
pub max_blob_bytes_per_item: usize,
pub max_blob_bytes_per_response: usize,
}
impl Default for EvalOptions {
fn default() -> Self {
EvalOptions {
only: None,
lease: std::time::Duration::from_secs(300),
max_items: 100,
dry_run: false,
node: default_node_id(),
max_blob_bytes_per_item: MAX_BLOB_BYTES_PER_ITEM,
max_blob_bytes_per_response: MAX_BLOB_BYTES_PER_RESPONSE,
}
}
}
fn short(s: &str, n: usize) -> &str {
match s.char_indices().nth(n) {
Some((idx, _)) => &s[..idx],
None => s,
}
}
fn default_node_id() -> String {
let host = std::env::var("HOSTNAME")
.or_else(|_| std::env::var("COMPUTERNAME"))
.unwrap_or_else(|_| "node".into());
format!("{host}/{}", std::process::id())
}
#[derive(Debug, Clone, Default, Serialize)]
pub struct EvalReport {
pub claimed: usize,
pub skipped_not_due: usize,
pub skipped_locked: usize,
pub skipped_paused: usize,
pub items: usize,
pub duplicates: usize,
pub runs_started: usize,
pub ingested: usize,
pub unidentifiable: usize,
pub unusable: usize,
pub errors: Vec<String>,
pub draining: Vec<String>,
pub cursor_held: Vec<String>,
}
#[derive(Debug, Clone, Serialize)]
pub struct TriggerStatus {
pub trigger: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub name: Option<String>,
pub kind: String,
pub workflow: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub connector: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub connector_tool: Option<String>,
pub enabled: bool,
pub paused: bool,
pub due: bool,
pub leased_by: Option<String>,
pub next_due_at: Option<i64>,
pub last_fired_at: Option<i64>,
pub consecutive_failures: u32,
pub last_error: Option<String>,
pub exhausted: bool,
pub never_fired: bool,
#[serde(skip_serializing_if = "Option::is_none")]
pub unusable: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub cursor: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub op_cursor: Option<i64>,
pub chain_root: String,
}
pub fn trigger_name(trigger: &Trigger) -> Option<String> {
trigger
.common
.extra_fields
.get("name")
.and_then(|v| v.as_str())
.map(str::trim)
.filter(|s| !s.is_empty())
.map(str::to_string)
}
pub trait RunStarter: Send + Sync {
fn start(&self, workflow: &str, run_id: &str, input: serde_json::Value) -> StartResult;
}
#[derive(Debug, Clone, PartialEq)]
pub enum StartResult {
Started,
Duplicate,
Failed(String),
}
#[derive(Debug, Clone, Default)]
pub struct ConnectorCode {
pub allow: Vec<String>,
pub cache_dir: Option<std::path::PathBuf>,
pub sandbox_cmd: Option<String>,
pub timeout: Option<Option<std::time::Duration>>,
pub env: Option<areev_core::proc::EnvPolicy>,
pub db_locator: Option<String>,
}
struct PreparedConnector {
tool_hash: String,
code: areev_run::PreparedCode,
wants_blobs: bool,
}
fn normalize_address(uri: &str) -> String {
uri.trim()
.strip_prefix("cas://sha256:")
.unwrap_or_else(|| uri.trim())
.to_ascii_lowercase()
}
struct NoConnectorFallback;
impl HostToolExecutor for NoConnectorFallback {
fn execute(
&self,
tool_name: &str,
_hash: &str,
_input: &serde_json::Value,
_idem: &str,
) -> areev_run::ExecResult {
areev_run::ExecResult::Err {
cause: areev_run::FailCause::ExecutorError,
detail: format!(
"connector {tool_name:?} is declared as a grain, so it has no host command to \
fall back to"
),
}
}
}
pub struct Evaluator {
pub facade: Arc<AreevFacade>,
pub clock: Arc<dyn Clock>,
pub connector: Option<Arc<dyn HostToolExecutor>>,
pub connector_code: Option<ConnectorCode>,
pub starter: Option<Arc<dyn RunStarter>>,
pub credentials: std::collections::BTreeMap<String, areev_run::CredentialSource>,
pub ns: String,
pub principal: String,
}
impl Evaluator {
pub fn read_only(facade: Arc<AreevFacade>, clock: Arc<dyn Clock>, ns: &str) -> Evaluator {
Evaluator {
facade,
clock,
connector: None,
connector_code: None,
starter: None,
credentials: Default::default(),
ns: ns.to_string(),
principal: "user:local".into(),
}
}
pub fn declarations(&self) -> Result<Vec<(String, Trigger)>> {
let ns = self.ns.clone();
let grains = self
.facade
.with_store(|m| {
m.recent_live_scoped(&[ns], Some(GrainType::Trigger), MAX_TRIGGERS_SCANNED)
})
.map_err(|e| TriggerError::Storage { detail: e.to_string() })?;
let mut out = Vec::new();
for g in grains {
let hash = g.hash.to_hex();
match g.to_trigger() {
Ok(mut t) => {
if let Some(name) = g.get_str("name") {
t.common
.extra_fields
.insert("name".into(), serde_json::json!(name));
}
out.push((hash, t))
}
Err(e) => {
return Err(TriggerError::Malformed {
what: format!("declaration {hash} is unreadable: {e}"),
})
}
}
}
Ok(out)
}
pub fn status(&self) -> Result<Vec<TriggerStatus>> {
let now = self.clock.now_ms();
let mut out = Vec::new();
let mut chains: HashMap<String, Chain> = HashMap::new();
for (hash, t) in self.declarations()? {
let chain = self.resolve_chain(&hash, &mut chains)?;
let (st, _raw) = self.resolve_state(&chain, false)?;
let unusable = schedule::validate(&t).err().map(|e| e.to_string());
out.push(TriggerStatus {
trigger: hash,
name: trigger_name(&t),
kind: t.kind.as_str().to_string(),
workflow: t.workflow.clone(),
connector: t.connector.clone(),
connector_tool: t.connector_tool.clone(),
enabled: t.enabled,
paused: st.paused,
due: unusable.is_none()
&& t.enabled
&& !st.leased(now)
&& schedule::is_due(&t, &st, now).unwrap_or(false),
leased_by: st.leased(now).then(|| st.claimed_by.clone()).flatten(),
next_due_at: st.next_due_at,
last_fired_at: st.last_fired_at,
consecutive_failures: st.consecutive_failures,
last_error: st.last_error.clone(),
exhausted: st.exhausted,
never_fired: st.last_fired_at.is_none() && st.consecutive_failures == 0,
unusable,
cursor: st.cursor.clone(),
op_cursor: st.op_cursor,
chain_root: chain.root().to_string(),
});
}
Ok(out)
}
pub fn deliver(&self, trigger_hash: &str, payload: serde_json::Value) -> Result<EvalReport> {
let now = self.clock.now_ms();
let (hash, trigger) = self
.declarations()?
.into_iter()
.find(|(h, _)| h.starts_with(trigger_hash))
.ok_or_else(|| TriggerError::Malformed {
what: format!("no trigger matching '{trigger_hash}'"),
})?;
if !matches!(trigger.kind, TriggerKind::Webhook | TriggerKind::Manual) {
return Err(TriggerError::Malformed {
what: format!(
"trigger {} is a {} trigger — only webhook and manual triggers accept a \
delivery; a clocked or polling trigger fires from `trigger run`",
short(&hash, 12),
trigger.kind.as_str()
),
});
}
if !trigger.enabled {
return Err(TriggerError::Malformed {
what: format!("trigger {} is disabled", short(&hash, 12)),
});
}
let mut chains: HashMap<String, Chain> = HashMap::new();
let chain = self.resolve_chain(&hash, &mut chains)?;
let (state, _raw) = self.resolve_state(&chain, true)?;
if state.paused {
return Err(TriggerError::Malformed {
what: format!("trigger {} is paused", short(&hash, 12)),
});
}
let mut report = EvalReport::default();
let item = PollItem { id: String::new(), payload, blobs: Vec::new() };
report.items = 1;
let Some(value) = dedup_value(&item, &trigger.dedup_key) else {
report.unidentifiable = 1;
self.journal(
&hash,
&trigger,
&FireOutcome { items: 1, unidentifiable: 1, ..Default::default() },
Some("delivered"),
now,
)?;
return Ok(report);
};
let root = chain.root();
let run_id = run_id_for(root, trigger.connector.as_deref(), &value);
let legacy_run_ids: Vec<String> = chain
.legacy()
.iter()
.map(|h| run_id_for(h, trigger.connector.as_deref(), &value))
.collect();
match self.start_run(&trigger, &run_id, &legacy_run_ids, &item, &hash, &[]) {
StartOutcome::Started => report.runs_started = 1,
StartOutcome::Ingested => report.ingested = 1,
StartOutcome::Duplicate => report.duplicates = 1,
StartOutcome::Failed(why) => report.errors.push(why),
}
let outcome = FireOutcome {
items: 1,
runs_started: report.runs_started,
duplicates: report.duplicates,
..Default::default()
};
self.journal(&hash, &trigger, &outcome, Some("delivered"), now)?;
Ok(report)
}
pub fn run(&self, opts: &EvalOptions) -> Result<EvalReport> {
let mut report = EvalReport::default();
let now = self.clock.now_ms();
let mut chains: HashMap<String, Chain> = HashMap::new();
let declarations = self.declarations()?;
let (composites, members): (Vec<_>, Vec<_>) =
declarations.into_iter().partition(|(_, t)| t.kind == TriggerKind::Composite);
for (hash, trigger) in members.into_iter().chain(composites) {
if opts.only.as_ref().is_some_and(|only| only != &hash) {
continue;
}
if !trigger.enabled {
report.skipped_not_due += 1;
continue;
}
if let Err(why) = schedule::validate(&trigger) {
report.unusable += 1;
report.errors.push(format!("{}: {why}", short(&hash, 12)));
continue;
}
let chain = self.resolve_chain(&hash, &mut chains)?;
let (state, raw) = self.resolve_state(&chain, true)?;
if state.paused {
report.skipped_paused += 1;
continue;
}
if state.next_due_at.is_none() && !state.exhausted {
if let Some(first) = schedule::initial_due(&trigger, now)? {
let mut seeded = state.clone();
seeded.next_due_at = Some(first);
let _ = self
.facade
.with_store(|m| m.put_trigger_state(chain.root(), raw.as_deref(), &seeded))
.map_err(|e| TriggerError::Storage { detail: e.to_string() })?;
report.skipped_not_due += 1;
continue;
}
}
if !schedule::is_due(&trigger, &state, now)? {
report.skipped_not_due += 1;
continue;
}
if state.leased(now) {
report.skipped_locked += 1;
continue;
}
if opts.dry_run {
report.claimed += 1;
continue;
}
match self.fire(&hash, &chain, &trigger, state, raw.as_deref(), now, opts) {
Ok(outcome) if outcome.skipped_locked => report.skipped_locked += 1,
Ok(outcome) => {
if !outcome.fired_items.is_empty() {
if let Err(e) =
self.record_for_composites(&hash, &outcome.fired_items, now, &mut chains)
{
report.errors.push(e.to_string());
}
}
report.claimed += 1;
report.items += outcome.items;
report.duplicates += outcome.duplicates;
report.runs_started += outcome.runs_started;
report.ingested += outcome.ingested;
report.unidentifiable += outcome.unidentifiable;
for why in &outcome.failures {
report.errors.push(format!("{hash}: {why}"));
}
if !outcome.failures.is_empty() {
report.cursor_held.push(hash.clone());
}
if outcome.draining {
report.draining.push(hash.clone());
}
}
Err(e) => {
report.errors.push(e.to_string());
}
}
}
Ok(report)
}
#[allow(clippy::too_many_arguments)]
fn fire(
&self,
hash: &str,
chain: &Chain,
trigger: &Trigger,
state: TriggerState,
raw: Option<&str>,
now: i64,
opts: &EvalOptions,
) -> Result<FireOutcome> {
let root = chain.root();
let mut claimed = state.clone();
claimed.fence = claimed.fence.wrapping_add(1);
claimed.claimed_by = Some(opts.node.clone());
claimed.lease_until = Some(now + opts.lease.as_millis() as i64);
let won = self
.facade
.with_store(|m| m.put_trigger_state(root, raw, &claimed))
.map_err(|e| TriggerError::Storage { detail: e.to_string() })?;
if !won {
return Ok(FireOutcome { skipped_locked: true, ..Default::default() });
}
let claimed_raw = self
.facade
.with_store(|m| m.trigger_state(root))
.map_err(|e| TriggerError::Storage { detail: e.to_string() })?
.map(|(_, r)| r);
let result = self.collect_and_start(hash, chain, trigger, &claimed, now, opts);
let mut released = claimed.clone();
released.claimed_by = None;
released.lease_until = None;
match &result {
Ok(outcome) if outcome.failures.is_empty() => {
released.last_fired_at = Some(now);
released.consecutive_failures = 0;
released.last_error = None;
if let Some(c) = &outcome.cursor {
released.cursor = Some(c.clone());
}
if let Some(c) = outcome.op_cursor {
released.op_cursor = Some(c);
}
for k in &outcome.settled_keys {
released.partials.remove(k);
}
prune_partials(&mut released, trigger, now);
released.exhausted = schedule::exhausts_after(trigger, now);
released.next_due_at = if outcome.draining {
Some(now)
} else {
schedule::advance_after_firing(trigger, now, now)?
};
}
Ok(outcome) => {
released.last_fired_at = Some(now);
released.consecutive_failures = claimed.consecutive_failures.saturating_add(1);
released.last_error = Some(summarize_failures(&outcome.failures));
released.next_due_at =
Some(schedule::backoff_until(trigger, released.consecutive_failures, now));
}
Err(e) => {
released.consecutive_failures = claimed.consecutive_failures.saturating_add(1);
released.last_error = Some(e.to_string());
released.next_due_at =
Some(schedule::backoff_until(trigger, released.consecutive_failures, now));
}
}
let released_ok = self
.facade
.with_store(|m| m.put_trigger_state(root, claimed_raw.as_deref(), &released))
.map_err(|e| TriggerError::Storage { detail: e.to_string() })?;
if !released_ok {
self.journal(hash, trigger, &FireOutcome::default(), Some("claim lost"), now)?;
return Err(TriggerError::ClaimLost { trigger: hash.to_string() });
}
let outcome = result?;
self.journal(hash, trigger, &outcome, None, now)?;
Ok(outcome)
}
fn collect_and_start(
&self,
hash: &str,
chain: &Chain,
trigger: &Trigger,
state: &TriggerState,
now: i64,
opts: &EvalOptions,
) -> Result<FireOutcome> {
let mut outcome = FireOutcome::default();
let items: Vec<PollItem> = match trigger.kind {
TriggerKind::Polling => {
let response = self.poll(hash, trigger, state, opts)?;
outcome.cursor = response.cursor.clone();
outcome.draining = response.more;
if state.cursor.is_none() {
outcome.seeded = true;
return Ok(outcome);
}
response.items
}
TriggerKind::Interval | TriggerKind::Schedule | TriggerKind::Once => {
vec![PollItem {
id: now.to_string(),
payload: serde_json::json!({ "at_ms": now }),
blobs: Vec::new(),
}]
}
TriggerKind::Memory => {
outcome.seeded = state.op_cursor.is_none();
let (found, cursor) = self.scan_memory(hash, trigger, state, opts)?;
outcome.op_cursor = Some(cursor);
found
}
TriggerKind::Composite => {
let keys = self.satisfied_keys(hash, chain, trigger, now)?;
outcome.settled_keys = keys.clone();
keys.into_iter()
.map(|k| PollItem {
id: k.clone(),
payload: serde_json::json!({ "correlation": k }),
blobs: Vec::new(),
})
.collect()
}
TriggerKind::Webhook | TriggerKind::Manual => Vec::new(),
};
let mut items = items;
let item_refs = self.ingest_item_blobs(hash, &mut items, opts)?;
outcome.items = items.len();
for (item, refs) in items.into_iter().zip(item_refs) {
let Some(value) = dedup_value(&item, &trigger.dedup_key) else {
outcome.unidentifiable += 1;
continue;
};
outcome.fired_items.push(item.payload.clone());
let root = chain.root();
let run_id = run_id_for(root, trigger.connector.as_deref(), &value);
let legacy_run_ids: Vec<String> = chain
.legacy()
.iter()
.map(|h| run_id_for(h, trigger.connector.as_deref(), &value))
.collect();
match self.start_run(trigger, &run_id, &legacy_run_ids, &item, hash, &refs) {
StartOutcome::Started => outcome.runs_started += 1,
StartOutcome::Ingested => outcome.ingested += 1,
StartOutcome::Duplicate => outcome.duplicates += 1,
StartOutcome::Failed(why) => outcome.failures.push(why),
}
}
Ok(outcome)
}
fn record_for_composites(
&self,
member: &str,
items: &[serde_json::Value],
now: i64,
chains: &mut HashMap<String, Chain>,
) -> Result<()> {
for (chash, composite) in self.declarations()? {
let Some(alias) = composite.member_alias(member).map(str::to_string) else {
continue;
};
if composite.kind != TriggerKind::Composite {
continue;
}
let chain = self.resolve_chain(&chash, chains)?;
let (mut state, raw) = self.resolve_state(&chain, true)?;
for item in items {
let key = match &composite.correlate {
Some(ptr) => match item.pointer(ptr) {
Some(serde_json::Value::String(s)) => s.clone(),
Some(other) => other.to_string(),
None => continue,
},
None => String::new(),
};
let entry = state.partials.entry(key).or_insert_with(|| {
areev_store::PartialMatch { members: Default::default(), started_ms: now }
});
entry.members.insert(alias.clone(), now);
}
prune_partials(&mut state, &composite, now);
self.facade
.with_store(|m| m.put_trigger_state(chain.root(), raw.as_deref(), &state))
.map_err(|e| TriggerError::Storage { detail: e.to_string() })?;
}
Ok(())
}
fn satisfied_keys(
&self,
hash: &str,
chain: &Chain,
trigger: &Trigger,
now: i64,
) -> Result<Vec<String>> {
let predicate = trigger
.predicate
.as_ref()
.ok_or_else(|| TriggerError::Malformed {
what: format!("composite trigger {hash} has no predicate"),
})
.and_then(crate::predicate::from_value)?;
for referenced in crate::predicate::referenced_members(&predicate) {
if !trigger.members.contains_key(&referenced) {
return Err(TriggerError::UnknownMember {
trigger: hash.to_string(),
member: referenced,
});
}
}
let (state, _) = self
.facade
.with_store(|m| m.trigger_state(chain.root()))
.map_err(|e| TriggerError::Storage { detail: e.to_string() })?
.unwrap_or_default();
let mut out = Vec::new();
for (key, partial) in &state.partials {
if let Some(w) = trigger.window_ms {
if now - partial.started_ms > w {
continue;
}
}
let fired: Vec<String> = partial.members.keys().cloned().collect();
if crate::predicate::gate_satisfied(&predicate, &fired) {
out.push(key.clone());
}
}
Ok(out)
}
fn scan_memory(
&self,
hash: &str,
trigger: &Trigger,
state: &TriggerState,
opts: &EvalOptions,
) -> Result<(Vec<PollItem>, i64)> {
let predicate = trigger
.predicate
.as_ref()
.ok_or_else(|| TriggerError::Malformed {
what: format!("memory trigger {hash} has no predicate"),
})
.and_then(crate::predicate::from_value)?;
let Some(after) = state.op_cursor else {
let head = self
.facade
.with_store(|m| m.head_op_seq())
.map_err(|e| TriggerError::Storage { detail: e.to_string() })?;
return Ok((Vec::new(), head));
};
let ops = self
.facade
.with_store(|m| m.changes_since(after, opts.max_items))
.map_err(|e| TriggerError::Storage { detail: e.to_string() })?;
let mut items = Vec::new();
let mut cursor = after;
for op in ops {
cursor = cursor.max(op.op_seq);
if op.op == areev_store::OP_FORGET {
continue;
}
let Ok(grain) = self.facade.with_store(|m| m.get(&op.hash)) else {
continue;
};
if !crate::predicate::grain_matches(&grain, &predicate) {
continue;
}
let hex = op.hash.to_hex();
items.push(PollItem {
id: hex.clone(),
payload: serde_json::json!({
"hash": hex,
"grain_type": grain.grain_type.as_str(),
"op_seq": op.op_seq,
}),
blobs: Vec::new(),
});
}
Ok((items, cursor))
}
fn ingest_item_blobs(
&self,
hash: &str,
items: &mut [PollItem],
opts: &EvalOptions,
) -> Result<Vec<Vec<ContentRef>>> {
let violation = |detail: String| TriggerError::BlobContract {
trigger: short(hash, 12).to_string(),
detail,
};
let mut response_total = 0usize;
let mut all_refs = Vec::with_capacity(items.len());
for (idx, item) in items.iter_mut().enumerate() {
if item.blobs.is_empty() {
all_refs.push(Vec::new());
continue;
}
let mut item_total = 0usize;
let mut refs = Vec::with_capacity(item.blobs.len());
let mut uris = Vec::with_capacity(item.blobs.len());
for (bi, blob) in item.blobs.iter().enumerate() {
let name = blob.filename.as_deref().unwrap_or("unnamed");
let bytes = areev_core::b64::decode(&blob.b64).ok_or_else(|| {
violation(format!("item {idx} blob {bi} ({name}) is not valid base64"))
})?;
item_total += bytes.len();
response_total += bytes.len();
if item_total > opts.max_blob_bytes_per_item {
return Err(violation(format!(
"item {idx} blobs exceed the per-item budget ({} bytes decoded)",
opts.max_blob_bytes_per_item
)));
}
if response_total > opts.max_blob_bytes_per_response {
return Err(violation(format!(
"response blobs exceed the per-response budget ({} bytes decoded)",
opts.max_blob_bytes_per_response
)));
}
let uri = self
.facade
.with_store(|m| m.put_blob(&bytes))
.map_err(|e| TriggerError::Storage { detail: e.to_string() })?;
refs.push(ContentRef {
uri: uri.clone(),
modality: None,
mime_type: blob.mime.clone(),
size_bytes: Some(bytes.len() as u64),
checksum: Some(uri.trim_start_matches("cas://").to_string()),
metadata: blob.filename.as_ref().map(|f| serde_json::json!({ "filename": f })),
});
uris.push(uri);
}
rewrite_blob_refs(&mut item.payload, &uris)
.map_err(|d| violation(format!("item {idx}: {d}")))?;
item.blobs.clear();
all_refs.push(refs);
}
Ok(all_refs)
}
fn prepare_connector(
&self,
hash: &str,
trigger: &Trigger,
connector_name: &str,
) -> Result<Option<PreparedConnector>> {
let Some(reference) = trigger
.connector_tool
.as_deref()
.map(str::trim)
.filter(|r| !r.is_empty())
else {
return Ok(None);
};
let refuse = |detail: String| TriggerError::ConnectorCode {
trigger: hash.to_string(),
detail,
};
let Some(cfg) = self.connector_code.as_ref() else {
return Err(refuse(format!(
"connector {connector_name:?} is a grain ({reference}) but this host runs no \
code-carrying connectors — pass --allow-executor <address> (and --sandbox-cmd \
for a wasm module)"
)));
};
let hex = areev_core::types::strip_grain_scheme(reference);
let h = Hash::from_hex(hex).map_err(|e| {
refuse(format!("connector_tool {reference:?} is not a content address: {e}"))
})?;
let g = self
.facade
.with_store(|m| m.get(&h))
.map_err(|e| refuse(format!("connector Definition {hex} is unreadable: {e}")))?;
if g.grain_type != GrainType::Tool {
return Err(refuse(format!(
"connector_tool {hex} is a {:?} grain, not a Tool Definition",
g.grain_type
)));
}
let pin = areev_run::pin_from_definition(connector_name, &h, &g)
.map_err(|e| refuse(e.to_string()))?;
let Some(uri) = pin.executor_uri.clone() else {
return Err(refuse(format!(
"connector Definition {hex} names no executor_uri — a connector grain IS its \
code, so there is nothing to run"
)));
};
let addr = normalize_address(&uri);
if !cfg.allow.iter().any(|a| normalize_address(a) == addr) {
return Err(refuse(format!(
"connector {connector_name:?} names {uri}, which this host did not pin — \
add --allow-executor {addr}"
)));
}
if pin.runtime.as_deref().is_some_and(areev_run::is_sandbox_runtime)
&& cfg.sandbox_cmd.is_none()
{
return Err(refuse(format!(
"connector {connector_name:?} declares runtime {:?}, which needs the sandbox — \
pass --sandbox-cmd areev-sandbox",
pin.runtime.as_deref().unwrap_or_default()
)));
}
let bytes = self
.facade
.with_store(|m| m.get_blob(&uri))
.map_err(|e| refuse(format!("connector blob {uri} is unreadable: {e}")))?;
let wants_blobs = pin
.capabilities
.as_ref()
.and_then(|v| areev_run::Declaration::parse(v).ok())
.is_some_and(|d| d.declares_blob_read());
Ok(Some(PreparedConnector {
tool_hash: h.to_hex(),
code: areev_run::PreparedCode {
uri,
bytes,
runtime: pin.runtime,
limits: pin.runtime_limits,
capabilities: pin.capabilities,
},
wants_blobs,
}))
}
fn code_executor(&self, broker: &Arc<areev_run::Broker>) -> areev_run::CodeExecutor {
let cfg = self.connector_code.clone().unwrap_or_default();
let mut ce = areev_run::CodeExecutor::new(Arc::new(NoConnectorFallback));
for addr in &cfg.allow {
ce = ce.allow(addr);
}
if let Some(dir) = cfg.cache_dir {
ce = ce.cache_dir(dir);
}
if let Some(cmd) = cfg.sandbox_cmd.as_deref() {
ce = ce.sandbox_cmd(cmd);
}
if let Some(t) = cfg.timeout {
ce = ce.with_timeout(t);
}
if let Some(env) = cfg.env {
ce = ce.with_env_policy(env);
}
ce.with_egress(areev_run::EgressHandle::new(Arc::clone(broker)))
}
fn poll(
&self,
hash: &str,
trigger: &Trigger,
state: &TriggerState,
opts: &EvalOptions,
) -> Result<PollResponse> {
let connector_name = trigger.connector.as_deref().unwrap_or_default();
let prepared = self.prepare_connector(hash, trigger, connector_name)?;
if prepared.is_none() && self.connector.is_none() {
return Err(TriggerError::NoConnector {
trigger: hash.to_string(),
connector: connector_name.to_string(),
});
}
let policy = areev_run::EgressPolicy::from_config(trigger.config.as_ref())
.map_err(|what| TriggerError::Malformed { what })?;
let grant = self.credentials.keys().fold(
areev_run::CallerGrant::new()
.method("GET")
.method("POST")
.method("PUT")
.method("PATCH")
.method("DELETE"),
|g, name| g.credential(name),
);
let grants = areev_run::EgressGrants::new()
.default_for_all(grant.clone())
.grant(connector_name, grant);
let broker = Arc::new(
areev_run::Broker::start(policy, self.credentials.clone(), grants, "TRG-E009")
.map_err(|detail| TriggerError::Storage { detail })?,
);
if prepared.as_ref().is_some_and(|p| p.wants_blobs) {
let locator = self
.connector_code
.as_ref()
.and_then(|c| c.db_locator.as_deref())
.ok_or_else(|| TriggerError::ConnectorCode {
trigger: hash.to_string(),
detail: format!(
"connector {connector_name:?} declares {{\"blob\": {{\"read\": true}}}} \
but this host wired the evaluator no memory locator, so the broker \
cannot serve a blob read"
),
})?;
broker.serve_blobs(locator);
}
let request = PollRequest {
trigger: hash,
connector: connector_name,
scope: trigger.scope.as_deref(),
cursor: state.cursor.as_deref(),
max_items: opts.max_items,
config: trigger.config.as_ref(),
};
let payload = serde_json::to_value(&request).map_err(|e| TriggerError::ConnectorFailed {
trigger: hash.to_string(),
detail: format!("serialize request: {e}"),
})?;
let idem = format!("{hash}:{}", state.fence);
static EGRESS_ENV: std::sync::Mutex<()> = std::sync::Mutex::new(());
let result = match &prepared {
Some(p) => self.code_executor(&broker).execute_code(
connector_name,
&p.tool_hash,
&p.code,
&payload,
&idem,
),
None => {
let exec = self.connector.as_ref().expect("checked above");
let _guard = EGRESS_ENV.lock().unwrap_or_else(|e| e.into_inner());
std::env::set_var("AREEV_EGRESS_URL", broker.url());
if let Some(t) = broker.token_for(connector_name) {
std::env::set_var("AREEV_EGRESS_TOKEN", t);
}
let r = exec.execute(connector_name, hash, &payload, &idem);
std::env::remove_var("AREEV_EGRESS_URL");
std::env::remove_var("AREEV_EGRESS_TOKEN");
r
}
};
let refused = broker.refusals();
if !refused.is_empty() {
return Err(TriggerError::EgressRefused {
trigger: hash.to_string(),
host: refused
.iter()
.map(|r| format!("{} ({})", r.destination, r.reason))
.collect::<Vec<_>>()
.join(", "),
});
}
match result {
areev_run::ExecResult::Ok(v) => {
serde_json::from_value(v).map_err(|e| TriggerError::ConnectorFailed {
trigger: hash.to_string(),
detail: format!("response is not a poll response: {e}"),
})
}
areev_run::ExecResult::Err { detail, .. } => {
Err(TriggerError::ConnectorFailed { trigger: hash.to_string(), detail })
}
}
}
fn start_run(
&self,
trigger: &Trigger,
run_id: &str,
legacy_run_ids: &[String],
item: &PollItem,
trigger_hash: &str,
content_refs: &[ContentRef],
) -> StartOutcome {
let seen = self
.facade
.with_store(|m| m.run_grains(&self.ns, run_id, 0, 1))
.map(|g| !g.is_empty())
.unwrap_or(false);
if seen {
return StartOutcome::Duplicate;
}
for legacy_id in legacy_run_ids {
let seen_legacy = self
.facade
.with_store(|m| m.run_grains(&self.ns, legacy_id, 0, 1))
.map(|g| !g.is_empty())
.unwrap_or(false);
if seen_legacy {
return StartOutcome::Duplicate;
}
}
let build_event = || {
let mut event = Event::new(&item.payload.to_string())
.namespace(&self.ns)
.extra_field("trigger", serde_json::json!(trigger_hash))
.extra_field("run_id", serde_json::json!(run_id));
if let Some(c) = &trigger.connector {
event = event.extra_field("connector", serde_json::json!(c));
}
for cr in content_refs {
event = event.content_ref(cr.clone());
}
event
};
let Some(starter) = &self.starter else {
return match self.facade.with_store(|m| m.add(&build_event())) {
Ok(_) => StartOutcome::Ingested,
Err(e) => StartOutcome::Failed(format!("ingest: {e}")),
};
};
let mut input = serde_json::json!({
"trigger": trigger_hash,
"connector": trigger.connector,
"scope": trigger.scope,
"item": item.payload,
});
if let Some(spec) = &trigger.context_query {
match self.assemble_context(spec, &item.payload) {
Ok(ctx) => {
input["context"] = ctx;
}
Err(why) => {
return StartOutcome::Failed(format!("context_query \"{spec}\": {why}"));
}
}
}
match starter.start(trigger.workflow_hash(), run_id, input) {
StartResult::Started => {
let _ = self.facade.with_store(|m| m.add(&build_event()));
StartOutcome::Started
}
StartResult::Duplicate => {
let _ = self.facade.with_store(|m| m.add(&build_event()));
StartOutcome::Duplicate
}
StartResult::Failed(why) => StartOutcome::Failed(why),
}
}
fn assemble_context(
&self,
spec: &str,
payload: &serde_json::Value,
) -> std::result::Result<serde_json::Value, String> {
let spec = crate::context::parse_context_query(spec)?;
let mut bindings: Vec<(String, areev_cal::ast::Value)> = Vec::new();
for (param, pointer) in &spec.bindings {
let v = payload.pointer(pointer).ok_or_else(|| {
format!("parameter ${param}: pointer {pointer} does not resolve against the firing item")
})?;
let value = match v {
serde_json::Value::String(s) => {
areev_cal::ast::Value::String { value: s.clone() }
}
serde_json::Value::Number(n) => areev_cal::ast::Value::Number {
value: n.as_f64().ok_or_else(|| {
format!("parameter ${param}: {pointer} is not a representable number")
})?,
},
serde_json::Value::Bool(b) => areev_cal::ast::Value::Boolean { value: *b },
other => {
return Err(format!(
"parameter ${param}: pointer {pointer} must land on a scalar, found {}",
match other {
serde_json::Value::Null => "null",
serde_json::Value::Array(_) => "an array",
_ => "an object",
}
))
}
};
bindings.push((param.clone(), value));
}
let ex = areev_cal::CalExecutor::new(areev_cal::CalExecutorConfig {
allow_destructive_ops: false,
..areev_cal::CalExecutorConfig::default()
});
let query = areev_cal::ast::CalQuery {
version: Default::default(),
statement: areev_cal::ast::CalStatement::RunQuery(areev_cal::ast::RunQueryStmt {
name: spec.name.clone(),
bindings,
span: None,
}),
pipeline: Vec::new(),
with_options: Vec::new(),
format: None,
let_bindings: Vec::new(),
user_vars: Default::default(),
let_values: Default::default(),
warnings: Vec::new(),
};
let res = ex
.execute_parsed(query, &format!("RUN \"{}\"", spec.name), self.facade.as_ref())
.map_err(|e| e.to_string())?;
serde_json::to_value(&res.result).map_err(|e| e.to_string())
}
fn journal(
&self,
hash: &str,
trigger: &Trigger,
outcome: &FireOutcome,
note: Option<&str>,
now: i64,
) -> Result<()> {
let mut obs = Observation::new("areev:trigger", "trigger:firing")
.namespace(TRIGGER_NS)
.created_at(now)
.extra_field("trigger", serde_json::json!(hash))
.extra_field("kind", serde_json::json!(trigger.kind.as_str()))
.extra_field("workflow", serde_json::json!(trigger.workflow))
.extra_field("items", serde_json::json!(outcome.items))
.extra_field("runs_started", serde_json::json!(outcome.runs_started))
.extra_field("duplicates", serde_json::json!(outcome.duplicates));
if outcome.unidentifiable > 0 {
obs = obs.extra_field("unidentifiable", serde_json::json!(outcome.unidentifiable));
}
if outcome.ingested > 0 {
obs = obs.extra_field("ingested", serde_json::json!(outcome.ingested));
}
if !outcome.failures.is_empty() {
obs = obs.extra_field("failures", serde_json::json!(outcome.failures));
obs = obs.extra_field("cursor_held", serde_json::json!(true));
}
if outcome.seeded {
obs = obs.extra_field("seeded", serde_json::json!(true));
}
if let Some(n) = note {
obs = obs.extra_field("note", serde_json::json!(n));
}
self.facade
.with_store(|m| m.add_if_novel(&obs))
.map(|_| ())
.map_err(|e| TriggerError::Storage { detail: e.to_string() })
}
fn resolve_chain(&self, hash: &str, cache: &mut HashMap<String, Chain>) -> Result<Chain> {
if let Some(c) = cache.get(hash) {
return Ok(c.clone());
}
let h = Hash::from_hex(hash).map_err(|e| TriggerError::Storage { detail: e.to_string() })?;
let hashes: Vec<String> = self
.facade
.with_store(|m| m.supersession_chain(&h))
.map_err(|e| TriggerError::Storage { detail: e.to_string() })?
.into_iter()
.map(|hh| hh.to_hex())
.collect();
let chain = Chain(hashes);
cache.insert(hash.to_string(), chain.clone());
Ok(chain)
}
fn resolve_state(&self, chain: &Chain, adopt: bool) -> Result<(TriggerState, Option<String>)> {
let root = chain.root();
if let Some((state, raw)) = self
.facade
.with_store(|m| m.trigger_state(root))
.map_err(|e| TriggerError::Storage { detail: e.to_string() })?
{
return Ok((state, Some(raw)));
}
let mut best: Option<(&str, TriggerState)> = None;
for legacy_hash in chain.legacy() {
if let Some((state, _raw)) = self
.facade
.with_store(|m| m.trigger_state(legacy_hash))
.map_err(|e| TriggerError::Storage { detail: e.to_string() })?
{
let better = best
.as_ref()
.is_none_or(|(_, current)| state_is_more_advanced(&state, current));
if better {
best = Some((legacy_hash.as_str(), state));
}
}
}
let Some((legacy_hash, state)) = best else {
return Ok((TriggerState::default(), None));
};
if !adopt {
return Ok((state, None));
}
match self.facade.with_store(|m| m.put_trigger_state(root, None, &state)) {
Ok(_) => {}
Err(AreevError::ReadOnly(_)) => return Ok((state, None)),
Err(e) => return Err(TriggerError::Storage { detail: e.to_string() }),
}
if let Err(e) = self.facade.with_store(|m| m.clear_trigger_state(legacy_hash)) {
if !matches!(e, AreevError::ReadOnly(_)) {
return Err(TriggerError::Storage { detail: e.to_string() });
}
}
let (state, raw) = self
.facade
.with_store(|m| m.trigger_state(root))
.map_err(|e| TriggerError::Storage { detail: e.to_string() })?
.ok_or_else(|| TriggerError::Storage {
detail: format!(
"adopted trigger state for {root} vanished immediately after writing"
),
})?;
Ok((state, Some(raw)))
}
}
#[derive(Debug, Clone)]
struct Chain(Vec<String>);
impl Chain {
fn root(&self) -> &str {
self.0.last().map(String::as_str).expect("a chain always has at least its head")
}
fn legacy(&self) -> &[String] {
&self.0[..self.0.len() - 1]
}
}
fn state_is_more_advanced(candidate: &TriggerState, current_best: &TriggerState) -> bool {
(candidate.last_fired_at, candidate.op_cursor, candidate.cursor.is_some())
> (current_best.last_fired_at, current_best.op_cursor, current_best.cursor.is_some())
}
fn summarize_failures(failures: &[String]) -> String {
match failures {
[one] => one.clone(),
many => format!("{} item(s) failed to start: {}", many.len(), many.join("; ")),
}
}
fn prune_partials(state: &mut TriggerState, trigger: &Trigger, now: i64) {
let Some(window) = trigger.window_ms else {
return;
};
state.partials.retain(|_, p| now - p.started_ms <= window);
}
#[derive(Debug, Default, Clone)]
struct FireOutcome {
items: usize,
runs_started: usize,
ingested: usize,
duplicates: usize,
unidentifiable: usize,
failures: Vec<String>,
cursor: Option<String>,
fired_items: Vec<serde_json::Value>,
settled_keys: Vec<String>,
op_cursor: Option<i64>,
draining: bool,
seeded: bool,
skipped_locked: bool,
}
enum StartOutcome {
Started,
Ingested,
Duplicate,
Failed(String),
}
pub fn run_id_for(trigger_hash: &str, connector: Option<&str>, dedup_value: &str) -> String {
let mut h = Sha256::new();
h.update(b"areev-trigger:v1");
h.update([0x1f]);
h.update(connector.unwrap_or_default().as_bytes());
h.update([0x1f]);
h.update(dedup_value.as_bytes());
let digest = hex(&h.finalize());
format!("{}-{}", short(trigger_hash, 8), short(&digest, 12))
}
fn hex(bytes: &[u8]) -> String {
bytes.iter().map(|b| format!("{b:02x}")).collect()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_run_id_is_stable_for_the_same_item() {
let a = run_id_for("abcdef1234", Some("gmail"), "m1");
let b = run_id_for("abcdef1234", Some("gmail"), "m1");
assert_eq!(a, b, "the same item must always mint the same id");
}
#[test]
fn different_items_get_different_ids() {
let a = run_id_for("abcdef1234", Some("gmail"), "m1");
let b = run_id_for("abcdef1234", Some("gmail"), "m2");
assert_ne!(a, b);
}
#[test]
fn the_connector_is_part_of_the_identity() {
let a = run_id_for("abcdef1234", Some("gmail"), "1");
let b = run_id_for("abcdef1234", Some("stripe"), "1");
assert_ne!(a, b);
}
#[test]
fn the_id_names_its_trigger_for_a_human_reading_a_run_list() {
let id = run_id_for("abcdef1234567890", Some("gmail"), "m1");
assert!(id.starts_with("abcdef12"), "{id}");
assert!(id.len() < 32, "short enough to read: {id}");
}
#[test]
fn a_short_trigger_hash_does_not_panic() {
let id = run_id_for("ab", None, "x");
assert!(id.starts_with("ab-"));
}
}