areev_trigger/eval.rs
1//! The evaluation cycle: find due, claim, invoke, ingest, start, release.
2//!
3//! One invocation performs the whole cycle and exits. Nothing stays resident.
4//! Losing a claim race is a no-op exit, not an error — that is the steady state
5//! when several nodes share a heartbeat.
6//!
7//! ## Why correctness does not rest on the lease
8//!
9//! A lease deduplicates *"who is working right now"*. But the thing that
10//! actually needs deduplicating — a firing of trigger T for item I — has a
11//! natural identity, so the run id is derived from it. `areev run start`
12//! already refuses a duplicate run id, so two evaluators racing produce one run
13//! and one recorded skip, with no lease consulted at all: no duration to guess,
14//! no fencing token to compare, no clock-skew window.
15//!
16//! The lease then does one narrower job: stopping two nodes from both making
17//! the same expensive, side-effecting connector call. Losing it can cost a
18//! wasted API call; it can never cause a missed or duplicated firing. That is
19//! the risk profile a lease should carry.
20
21use std::collections::HashMap;
22use std::sync::Arc;
23
24use areev_cal::AreevFacade;
25use areev_core::error::{AreevError, Hash};
26use areev_core::types::{ContentRef, Event, Grain, GrainType, Observation, Trigger, TriggerKind};
27use areev_run::HostToolExecutor;
28use areev_store::TriggerState;
29use serde::Serialize;
30use sha2::{Digest, Sha256};
31
32use crate::clock::Clock;
33use crate::connector::{
34 dedup_value, rewrite_blob_refs, PollItem, PollRequest, PollResponse,
35 MAX_BLOB_BYTES_PER_ITEM, MAX_BLOB_BYTES_PER_RESPONSE,
36};
37use crate::error::{Result, TriggerError};
38use crate::schedule;
39
40/// Namespace for trigger firing records. Alongside the runtime's own
41/// `agent:harness`, so operational journalling stays out of user namespaces.
42pub const TRIGGER_NS: &str = "agent:triggers";
43
44/// How many triggers one invocation will look at. A backstop, not a policy.
45const MAX_TRIGGERS_SCANNED: usize = 4096;
46
47/// Knobs for one evaluation pass.
48#[derive(Debug, Clone)]
49pub struct EvalOptions {
50 /// Evaluate only this trigger (its content address).
51 pub only: Option<String>,
52 /// How long a claim is held before another evaluator may take over.
53 /// Deliberately generous: a lease shorter than the worst-case connector
54 /// runtime causes takeovers mid-poll, which costs duplicate API calls.
55 pub lease: std::time::Duration,
56 /// Cap on items requested per firing.
57 pub max_items: usize,
58 /// Report what would happen and touch nothing. The safe first command on a
59 /// new deployment.
60 pub dry_run: bool,
61 /// This evaluator's identity, recorded in the claim.
62 pub node: String,
63 /// Decoded-size budget for one item's connector blobs (#93). Overrun
64 /// refuses the whole poll (TRG-E011) rather than truncating.
65 pub max_blob_bytes_per_item: usize,
66 /// Decoded-size budget for one poll response's blobs (#93).
67 pub max_blob_bytes_per_response: usize,
68}
69
70impl Default for EvalOptions {
71 fn default() -> Self {
72 EvalOptions {
73 only: None,
74 lease: std::time::Duration::from_secs(300),
75 max_items: 100,
76 dry_run: false,
77 node: default_node_id(),
78 max_blob_bytes_per_item: MAX_BLOB_BYTES_PER_ITEM,
79 max_blob_bytes_per_response: MAX_BLOB_BYTES_PER_RESPONSE,
80 }
81 }
82}
83
84/// First `n` characters, safely — `&s[..n]` panics on a short string or a
85/// multi-byte boundary, and a message-formatting helper must not be what takes
86/// the process down. Declarations can arrive by bundle import from an
87/// implementation we did not write.
88fn short(s: &str, n: usize) -> &str {
89 match s.char_indices().nth(n) {
90 Some((idx, _)) => &s[..idx],
91 None => s,
92 }
93}
94
95/// A stable-ish identity for this evaluator. Host name plus pid: enough to tell
96/// two nodes apart in a claim, and to recognise our own stale claim after a
97/// restart.
98fn default_node_id() -> String {
99 let host = std::env::var("HOSTNAME")
100 .or_else(|_| std::env::var("COMPUTERNAME"))
101 .unwrap_or_else(|_| "node".into());
102 format!("{host}/{}", std::process::id())
103}
104
105/// What one evaluation pass did. Shaped like `loop run`'s report for symmetry.
106#[derive(Debug, Clone, Default, Serialize)]
107pub struct EvalReport {
108 pub claimed: usize,
109 pub skipped_not_due: usize,
110 pub skipped_locked: usize,
111 pub skipped_paused: usize,
112 pub items: usize,
113 pub duplicates: usize,
114 pub runs_started: usize,
115 /// Items recorded without starting a workflow, because no runtime is wired
116 /// in. Counted apart from `runs_started` so the report never claims a run
117 /// happened when none did.
118 pub ingested: usize,
119 /// Items the connector returned that carry no usable identity, so no run
120 /// could be minted for them. Reported rather than silently dropped: an item
121 /// nobody can name is a connector bug, and it should be visible as one.
122 pub unidentifiable: usize,
123 /// Declarations that cannot fire as written — a cron that does not parse, a
124 /// timezone this build refuses, a composite gate naming a member the
125 /// declaration does not carry.
126 ///
127 /// Counted APART from `skipped_not_due` on purpose. Folded in there, an
128 /// unusable trigger is indistinguishable from a healthy one waiting its
129 /// turn, which is the failure mode with no symptom: the work simply never
130 /// happens and every report looks green. The reason lands in `errors`.
131 pub unusable: usize,
132 /// Not fatal to the pass: one broken trigger must not stop the others.
133 pub errors: Vec<String>,
134 /// A connector reported a backlog, so this trigger is due again at once
135 /// rather than after its interval.
136 pub draining: Vec<String>,
137 /// Triggers whose cursor was deliberately left unmoved this pass because
138 /// at least one item in the firing failed to start a run (#129) — the
139 /// Event add failed, declared context was unavailable, or the run
140 /// itself refused. The same page is offered again next tick; whatever
141 /// already started comes back as a `duplicates` count rather than
142 /// double-processing, which is what makes re-polling safe — see #128's
143 /// stable run ids, a prerequisite for this.
144 pub cursor_held: Vec<String>,
145}
146
147/// One trigger considered, for `--dry-run` and `status`.
148#[derive(Debug, Clone, Serialize)]
149pub struct TriggerStatus {
150 pub trigger: String,
151 /// The declaration's `name`, when it carries one.
152 ///
153 /// `name` is what a human uses to identify a trigger, and it is accepted
154 /// on write — but nothing read it back (#73), so identity fell onto the
155 /// workflow hash, which is stable only until the plan is re-declared. It
156 /// rides in `extra_fields` rather than being a typed field: OMS §A.7 has
157 /// no Trigger `name`, and inventing one would be a spec-level decision
158 /// for a label.
159 #[serde(skip_serializing_if = "Option::is_none")]
160 pub name: Option<String>,
161 pub kind: String,
162 pub workflow: String,
163 /// The connector's name, and — when its code is a grain (#185) — the
164 /// Definition that carries it.
165 ///
166 /// Reported because a reader looking at a heartbeat needs to know which of
167 /// the two connector kinds this is: the answer decides whether the code
168 /// travelled with the memory or lives on whichever machine happens to run
169 /// the pass, and that is the first question a stale-poll investigation
170 /// asks.
171 #[serde(skip_serializing_if = "Option::is_none")]
172 pub connector: Option<String>,
173 #[serde(skip_serializing_if = "Option::is_none")]
174 pub connector_tool: Option<String>,
175 pub enabled: bool,
176 pub paused: bool,
177 pub due: bool,
178 pub leased_by: Option<String>,
179 pub next_due_at: Option<i64>,
180 pub last_fired_at: Option<i64>,
181 pub consecutive_failures: u32,
182 pub last_error: Option<String>,
183 /// Will never fire again — a one-shot past its instant.
184 pub exhausted: bool,
185 /// Never fired and no failure recorded — the state an unnoticed
186 /// misconfiguration sits in, so it is reported rather than inferred.
187 pub never_fired: bool,
188 /// The plan this trigger names has been EDITED: the current head of its
189 /// supersession chain, when that is not the hash the trigger points at.
190 ///
191 /// A Workflow is content-addressed, so editing one mints a new address and
192 /// a trigger keeps starting the version it was declared against. That is
193 /// the correct default — a plan edit must not silently change what fires
194 /// unattended — but it is invisible without this: nothing in a listing
195 /// distinguished a trigger running the current plan from one running a
196 /// version three edits old, and the run itself succeeds either way.
197 ///
198 /// Reported, never followed. `areev trigger retarget` is the deliberate
199 /// act that moves it, and it is a human's to take.
200 #[serde(skip_serializing_if = "Option::is_none")]
201 pub plan_head: Option<String>,
202 /// Why this declaration can never fire, if it cannot.
203 ///
204 /// A trigger can become unusable without ever being written through a
205 /// validating path — it can arrive by bundle import from an implementation
206 /// that validated differently, or predate the check. So the evaluator
207 /// reports it rather than assuming the write path caught everything.
208 #[serde(skip_serializing_if = "Option::is_none")]
209 pub unusable: Option<String>,
210 /// The connector's last reported position, for a trigger that has ever
211 /// polled. Opaque and connector-defined (#128) — surfaced so an operator
212 /// can SEE the state that a re-pointed trigger's evaluation carries
213 /// forward, which before this was invisible even though nothing else
214 /// could carry it by hand.
215 #[serde(skip_serializing_if = "Option::is_none")]
216 pub cursor: Option<String>,
217 /// The op-log position, a `memory` trigger's cursor equivalent.
218 #[serde(skip_serializing_if = "Option::is_none")]
219 pub op_cursor: Option<i64>,
220 /// The hash evaluation state is actually keyed under: `trg:<chain_root>`
221 /// (#128). Equal to `trigger` for a trigger that has never been
222 /// superseded — the common case, unchanged from before this field
223 /// existed. Differs only for a trigger re-pointed by `SUPERSEDE`, where
224 /// it names the ORIGINAL declaration's hash — the identity the cursor
225 /// and dedup fence actually survive under.
226 pub chain_root: String,
227}
228
229/// The declaration's human label, if it has one.
230///
231/// Written through `extra_fields` by every surface that accepts `name` on a
232/// trigger, and read back here so `list` and `status` can print it (#73).
233/// Blank is treated as absent: a name that renders as nothing is worse than
234/// none, because it displaces the hash a reader could otherwise have used.
235pub fn trigger_name(trigger: &Trigger) -> Option<String> {
236 trigger
237 .common
238 .extra_fields
239 .get("name")
240 .and_then(|v| v.as_str())
241 .map(str::trim)
242 .filter(|s| !s.is_empty())
243 .map(str::to_string)
244}
245
246/// Starts the workflow a trigger is bound to.
247///
248/// A seam rather than a direct `areev_run::Runner` call, for the same reason
249/// the CAL executor takes a governance host: the runtime sits above this crate
250/// in the dependency order for hosts that want it, and a test needs to observe
251/// what *would* be started without spending a real run. It is also where the
252/// duplicate rule lives, which is the whole idempotency story.
253pub trait RunStarter: Send + Sync {
254 fn start(&self, workflow: &str, run_id: &str, input: serde_json::Value) -> StartResult;
255}
256
257/// What starting a run did.
258#[derive(Debug, Clone, PartialEq)]
259pub enum StartResult {
260 Started,
261 /// A run with this id already exists. **Not an error** — it is the
262 /// idempotency guarantee working: this item was already handled, by an
263 /// earlier firing or by another node that won the race.
264 Duplicate,
265 Failed(String),
266}
267
268/// How this host runs a connector that is a **grain** rather than a command
269/// (#185).
270///
271/// Host config, never in the file, for exactly the reason `--allow-executor`
272/// is host config on the run path: a bundle carries the connector's code, so a
273/// permission arriving in the same bundle as the code it authorizes is not a
274/// permission. A trigger may name any Definition it likes; nothing runs unless
275/// this host pinned its address.
276///
277/// Absent (`None` on the evaluator) means a trigger naming a connector
278/// Definition refuses with `TRG-E012` — the same fail-closed shape as a due
279/// polling trigger with no `--connector-cmd`.
280#[derive(Debug, Clone, Default)]
281pub struct ConnectorCode {
282 /// Content addresses this host will execute — `--allow-executor`, the
283 /// same comma list and the same meaning the run path gives it.
284 pub allow: Vec<String>,
285 /// Where a pinned blob is materialized (`--executor-cache`).
286 pub cache_dir: Option<std::path::PathBuf>,
287 /// The `areev-sandbox` runner, argv-split by the executor. Required for a
288 /// `wasm32-areev`/`wasm32-areev-io` connector, which is the shape this
289 /// exists for; a native pinned blob runs without one.
290 pub sandbox_cmd: Option<String>,
291 /// Per-invocation wall-clock ceiling (`--executor-timeout`), in the run
292 /// path's tri-state: `None` leaves the executor's own default standing,
293 /// `Some(None)` waits forever on request, `Some(Some(d))` overrides.
294 pub timeout: Option<Option<std::time::Duration>>,
295 /// Environment policy for a NATIVE pinned blob. The sandbox clears its
296 /// environment regardless — see `CodeExecutor`.
297 pub env: Option<areev_core::proc::EnvPolicy>,
298 /// The memory's locator, so a connector declaring `{"blob": {"read":
299 /// true}}` can be served by the per-poll broker. The evaluator holds the
300 /// memory while a connector runs, and the broker's blob read deliberately
301 /// does not open it — it goes to the `.blobs` sidecar, or to one
302 /// short-lived Postgres connection of its own (#202).
303 pub db_locator: Option<String>,
304}
305
306/// A connector Definition resolved, pinned and read — everything
307/// `execute_code` needs, gathered before a broker is started so a refusal
308/// costs no side effects.
309struct PreparedConnector {
310 /// The Definition's own content address, which is what the trigger names.
311 tool_hash: String,
312 code: areev_run::PreparedCode,
313 /// True when the pinned declaration asks to read CAS blobs, which is what
314 /// decides whether the per-poll broker serves them.
315 wants_blobs: bool,
316 /// The Definition's `config` (#231) — the connector's WIRING, as opposed
317 /// to the trigger's `config`, which is the instance. A generic blob like
318 /// `rest.poll` is made provider-specific here, beside the `capabilities`
319 /// block it has to agree with, and travels with the code rather than
320 /// with each declaration that names it.
321 config: Option<serde_json::Value>,
322}
323
324/// The `config` a connector is handed: the Definition's wiring with the
325/// trigger's own keys laid over it.
326///
327/// Shallow, and the trigger wins — the more specific instance specializes the
328/// shared declaration, which is what lets two mailboxes share one Definition.
329/// Either side alone passes through untouched, so a connector that predates
330/// Definition config sees exactly what it saw before.
331fn merged_config(
332 definition: Option<&serde_json::Value>,
333 trigger: Option<&serde_json::Value>,
334) -> Option<serde_json::Value> {
335 match (definition, trigger) {
336 (None, t) => t.cloned(),
337 (Some(d), None) => Some(d.clone()),
338 (Some(d), Some(t)) => match (d.as_object(), t.as_object()) {
339 (Some(d), Some(t)) => {
340 let mut out = d.clone();
341 for (k, v) in t {
342 out.insert(k.clone(), v.clone());
343 }
344 Some(serde_json::Value::Object(out))
345 }
346 // Neither is an object: nothing to merge key-wise, and the
347 // instance is the more specific statement.
348 _ => Some(t.clone()),
349 },
350 }
351}
352
353/// `cas://sha256:<hex>` and a bare `<hex>` are the same pin, compared the way
354/// the executor compares them.
355fn normalize_address(uri: &str) -> String {
356 uri.trim()
357 .strip_prefix("cas://sha256:")
358 .unwrap_or_else(|| uri.trim())
359 .to_ascii_lowercase()
360}
361
362/// The inner executor a connector's [`areev_run::CodeExecutor`] wraps.
363///
364/// It is never reached: this path only ever calls `execute_code`, which
365/// dispatches the blob itself. It exists because `CodeExecutor` wraps
366/// something by construction, and what it wraps on the run path — the host's
367/// `--tool-cmd` — must NOT be silently reachable here. A connector that
368/// resolved to a command would be a different program than the declaration
369/// names.
370struct NoConnectorFallback;
371
372impl HostToolExecutor for NoConnectorFallback {
373 fn execute(
374 &self,
375 tool_name: &str,
376 _hash: &str,
377 _input: &serde_json::Value,
378 _idem: &str,
379 ) -> areev_run::ExecResult {
380 areev_run::ExecResult::Err {
381 cause: areev_run::FailCause::ExecutorError,
382 detail: format!(
383 "connector {tool_name:?} is declared as a grain, so it has no host command to \
384 fall back to"
385 ),
386 }
387 }
388}
389
390/// Evaluates triggers against one memory.
391pub struct Evaluator {
392 pub facade: Arc<AreevFacade>,
393 pub clock: Arc<dyn Clock>,
394 /// Executes connector commands. `None` means a due polling trigger fails
395 /// loudly with `TRG-E003` rather than quietly doing nothing — a poll that
396 /// silently returns no items is indistinguishable from a healthy source.
397 pub connector: Option<Arc<dyn HostToolExecutor>>,
398 /// How a connector that is a GRAIN runs (#185). `None` means this host
399 /// runs host-command connectors only, and a trigger naming a connector
400 /// Definition refuses with `TRG-E012` rather than falling back to
401 /// `--connector-cmd` — a fallback would run a different program than the
402 /// declaration names, which is the failure with no symptom the pin exists
403 /// to refuse.
404 pub connector_code: Option<ConnectorCode>,
405 /// Starts the bound workflow. `None` ingests the item and records the
406 /// firing without starting anything — what `--no-start` is for, and what
407 /// the ingest-only tests use.
408 pub starter: Option<Arc<dyn RunStarter>>,
409 /// Credentials the broker may attach, by name. Values live here and never
410 /// in a grain or a connector's environment: a declaration names a
411 /// credential, it never carries one.
412 /// A SOURCE per name, not a resolved value (#113): the heartbeat is the
413 /// path that most needs short-lived credentials, because it is the one
414 /// nobody is watching. An env-var source resolves once, as before; a
415 /// `cmd:`/`vault:` source is minted per TTL window inside the broker, so a
416 /// cadence that outlives a token's hour does not start 401'ing overnight.
417 pub credentials: std::collections::BTreeMap<String, areev_run::CredentialSource>,
418 pub ns: String,
419 pub principal: String,
420}
421
422impl Evaluator {
423 /// An evaluator that can inspect but not act: no connector, no runtime, no
424 /// credentials. What `list`, `show` and `status` want, and the shape that
425 /// makes "reading cannot fire anything" true by construction rather than by
426 /// remembering to pass `None` four times.
427 pub fn read_only(facade: Arc<AreevFacade>, clock: Arc<dyn Clock>, ns: &str) -> Evaluator {
428 Evaluator {
429 facade,
430 clock,
431 connector: None,
432 connector_code: None,
433 starter: None,
434 credentials: Default::default(),
435 ns: ns.to_string(),
436 principal: "user:local".into(),
437 }
438 }
439
440 /// Every trigger declaration in the memory, newest first.
441 pub fn declarations(&self) -> Result<Vec<(String, Trigger)>> {
442 let ns = self.ns.clone();
443 let grains = self
444 .facade
445 .with_store(|m| {
446 m.recent_live_scoped(&[ns], Some(GrainType::Trigger), MAX_TRIGGERS_SCANNED)
447 })
448 .map_err(|e| TriggerError::Storage { detail: e.to_string() })?;
449
450 let mut out = Vec::new();
451 for g in grains {
452 let hash = g.hash.to_hex();
453 match g.to_trigger() {
454 Ok(mut t) => {
455 // `to_trigger` reconstructs the typed fields; `extra_fields`
456 // is not restored for any grain type, so the declaration's
457 // `name` — an extra field by design, since OMS §A.7 has no
458 // Trigger `name` — has to be carried over here. Doing it in
459 // the shared deserializer would silently change what every
460 // read-modify-write re-serializes, which is the kind of
461 // change that moves content addresses.
462 if let Some(name) = g.get_str("name") {
463 t.common
464 .extra_fields
465 .insert("name".into(), serde_json::json!(name));
466 }
467 out.push((hash, t))
468 }
469 // A declaration this build cannot read is reported, never
470 // skipped silently: silence is the symptom of every trigger
471 // failure, so it must never also be the symptom of a bug.
472 Err(e) => {
473 return Err(TriggerError::Malformed {
474 what: format!("declaration {hash} is unreadable: {e}"),
475 })
476 }
477 }
478 }
479 Ok(out)
480 }
481
482 /// What every trigger's state looks like right now.
483 pub fn status(&self) -> Result<Vec<TriggerStatus>> {
484 let now = self.clock.now_ms();
485 let mut out = Vec::new();
486 // One cache for this call, same as a `run()` pass — a chain is
487 // walked at most once even though `declarations()` above may list
488 // the same workflow's triggers repeatedly.
489 let mut chains: HashMap<String, Chain> = HashMap::new();
490 for (hash, t) in self.declarations()? {
491 let chain = self.resolve_chain(&hash, &mut chains)?;
492 // Read-only: `adopt = false`. `trigger status`/`show` must not
493 // have the side effect of migrating a legacy state row — a real
494 // evaluation pass (`run`/`deliver`) does that; this only
495 // reports whatever it finds, root row or legacy.
496 let (st, _raw) = self.resolve_state(&chain, false)?;
497 let unusable = schedule::validate(&t).err().map(|e| e.to_string());
498 let plan_head = self.plan_head_if_stale(&t)?;
499 out.push(TriggerStatus {
500 trigger: hash,
501 name: trigger_name(&t),
502 kind: t.kind.as_str().to_string(),
503 workflow: t.workflow.clone(),
504 connector: t.connector.clone(),
505 connector_tool: t.connector_tool.clone(),
506 enabled: t.enabled,
507 paused: st.paused,
508 // An unusable declaration is never due. Saying otherwise would
509 // promise a firing that cannot happen.
510 due: unusable.is_none()
511 && t.enabled
512 && !st.leased(now)
513 && schedule::is_due(&t, &st, now).unwrap_or(false),
514 leased_by: st.leased(now).then(|| st.claimed_by.clone()).flatten(),
515 next_due_at: st.next_due_at,
516 last_fired_at: st.last_fired_at,
517 consecutive_failures: st.consecutive_failures,
518 last_error: st.last_error.clone(),
519 exhausted: st.exhausted,
520 never_fired: st.last_fired_at.is_none() && st.consecutive_failures == 0,
521 unusable,
522 cursor: st.cursor.clone(),
523 op_cursor: st.op_cursor,
524 plan_head,
525 chain_root: chain.root().to_string(),
526 });
527 }
528 Ok(out)
529 }
530
531 /// Ingest a payload delivered by the host for a `webhook` or `manual`
532 /// trigger.
533 ///
534 /// Areev never opens a port. The host already terminates TLS and
535 /// authenticates the sender — it is far better at both than a memory engine
536 /// would be — and hands the payload to a one-shot invocation. That keeps the
537 /// useful half of webhooks without a listening daemon.
538 ///
539 /// Idempotent on the same terms as a poll: the payload's dedup value mints
540 /// the run id, so a webhook delivered twice (which every provider does)
541 /// produces one run and one recorded skip.
542 pub fn deliver(&self, trigger_hash: &str, payload: serde_json::Value) -> Result<EvalReport> {
543 let now = self.clock.now_ms();
544 let (hash, trigger) = self
545 .declarations()?
546 .into_iter()
547 .find(|(h, _)| h.starts_with(trigger_hash))
548 .ok_or_else(|| TriggerError::Malformed {
549 what: format!("no trigger matching '{trigger_hash}'"),
550 })?;
551
552 if !matches!(trigger.kind, TriggerKind::Webhook | TriggerKind::Manual) {
553 return Err(TriggerError::Malformed {
554 what: format!(
555 "trigger {} is a {} trigger — only webhook and manual triggers accept a \
556 delivery; a clocked or polling trigger fires from `trigger run`",
557 short(&hash, 12),
558 trigger.kind.as_str()
559 ),
560 });
561 }
562 if !trigger.enabled {
563 return Err(TriggerError::Malformed {
564 what: format!("trigger {} is disabled", short(&hash, 12)),
565 });
566 }
567 // #128 — a single-trigger cache: `deliver` fires exactly one
568 // trigger, so this walks its chain at most once regardless of how
569 // many places below need the root.
570 let mut chains: HashMap<String, Chain> = HashMap::new();
571 let chain = self.resolve_chain(&hash, &mut chains)?;
572 let (state, _raw) = self.resolve_state(&chain, true)?;
573 if state.paused {
574 return Err(TriggerError::Malformed {
575 what: format!("trigger {} is paused", short(&hash, 12)),
576 });
577 }
578
579 let mut report = EvalReport::default();
580 let item = PollItem { id: String::new(), payload, blobs: Vec::new() };
581 report.items = 1;
582 let Some(value) = dedup_value(&item, &trigger.dedup_key) else {
583 report.unidentifiable = 1;
584 // Journal before returning. A delivery naming nothing is precisely
585 // the case the record exists for: the sender's payload shape
586 // changed under us, and stdout of whatever invoked this is not an
587 // audit record -- the replicating Observation is.
588 self.journal(
589 &hash,
590 &trigger,
591 &FireOutcome { items: 1, unidentifiable: 1, ..Default::default() },
592 Some("delivered"),
593 now,
594 )?;
595 return Ok(report);
596 };
597 // Keyed on the chain ROOT (#128), so a delivery replayed after the
598 // trigger has been re-pointed by `SUPERSEDE` still mints the same id
599 // it always would have. `legacy_run_ids` covers a trigger already
600 // superseded before this fix — the id it would have minted under
601 // its old (then-current) head.
602 let root = chain.root();
603 let run_id = run_id_for(root, trigger.connector.as_deref(), &value);
604 let legacy_run_ids: Vec<String> = chain
605 .legacy()
606 .iter()
607 .map(|h| run_id_for(h, trigger.connector.as_deref(), &value))
608 .collect();
609 match self.start_run(&trigger, &run_id, &legacy_run_ids, &item, &hash, &[]) {
610 StartOutcome::Started => report.runs_started = 1,
611 StartOutcome::Ingested => report.ingested = 1,
612 StartOutcome::Duplicate => report.duplicates = 1,
613 StartOutcome::Failed(why) => report.errors.push(why),
614 }
615
616 let outcome = FireOutcome {
617 items: 1,
618 runs_started: report.runs_started,
619 duplicates: report.duplicates,
620 ..Default::default()
621 };
622 self.journal(&hash, &trigger, &outcome, Some("delivered"), now)?;
623 Ok(report)
624 }
625
626 /// Run one evaluation pass.
627 pub fn run(&self, opts: &EvalOptions) -> Result<EvalReport> {
628 let mut report = EvalReport::default();
629 let now = self.clock.now_ms();
630 // #128 — resolved once per trigger per PASS, never once per item:
631 // every chain walk below goes through this cache, shared with
632 // `record_for_composites` (a composite may be touched by several
633 // members firing in the same pass).
634 let mut chains: HashMap<String, Chain> = HashMap::new();
635
636 let declarations = self.declarations()?;
637 // Composites are settled after their members, so a gate can be
638 // satisfied in the same pass that completes it rather than waiting a
639 // whole heartbeat. Declaration order is newest-first and says nothing
640 // about dependency, so the split is explicit.
641 let (composites, members): (Vec<_>, Vec<_>) =
642 declarations.into_iter().partition(|(_, t)| t.kind == TriggerKind::Composite);
643
644 for (hash, trigger) in members.into_iter().chain(composites) {
645 if opts.only.as_ref().is_some_and(|only| only != &hash) {
646 continue;
647 }
648 if !trigger.enabled {
649 report.skipped_not_due += 1;
650 continue;
651 }
652 // Checked BEFORE dueness, so an unusable declaration is reported as
653 // what it is rather than disappearing into `not due` — where it
654 // looks exactly like a healthy trigger waiting its turn.
655 if let Err(why) = schedule::validate(&trigger) {
656 report.unusable += 1;
657 report.errors.push(format!("{}: {why}", short(&hash, 12)));
658 continue;
659 }
660 // #128 — every read/write of this trigger's evaluation state
661 // below is keyed on the chain ROOT, not on `hash` (the
662 // declaration's current head): a trigger re-pointed by
663 // `SUPERSEDE` mints a new head for the same standing rule, and
664 // keying state on the head alone would orphan the cursor and
665 // the dedup fence on every applied recommendation. For a
666 // never-superseded trigger root == hash, so this resolves to
667 // exactly the lookup this crate always did — a byte-identical
668 // no-op (pinned by `a_never_superseded_trigger_keys_state_on_its_own_hash`).
669 // `adopt = true`: this is a real evaluation pass with write
670 // access, so a legacy (pre-#128, head-keyed) row found along the
671 // way is migrated under the root.
672 let chain = self.resolve_chain(&hash, &mut chains)?;
673 let (state, raw) = self.resolve_state(&chain, true)?;
674
675 if state.paused {
676 report.skipped_paused += 1;
677 continue;
678 }
679 // An absolute schedule seen for the first time: record when it
680 // will be due rather than re-deriving it every pass, so `trigger
681 // status` can say when it next fires and the comparison afterwards
682 // is a plain timestamp check.
683 if state.next_due_at.is_none() && !state.exhausted {
684 if let Some(first) = schedule::initial_due(&trigger, now)? {
685 let mut seeded = state.clone();
686 seeded.next_due_at = Some(first);
687 let _ = self
688 .facade
689 .with_store(|m| m.put_trigger_state(chain.root(), raw.as_deref(), &seeded))
690 .map_err(|e| TriggerError::Storage { detail: e.to_string() })?;
691 report.skipped_not_due += 1;
692 continue;
693 }
694 }
695 if !schedule::is_due(&trigger, &state, now)? {
696 report.skipped_not_due += 1;
697 continue;
698 }
699 if state.leased(now) {
700 // The normal steady state with several nodes on one heartbeat,
701 // not a problem.
702 report.skipped_locked += 1;
703 continue;
704 }
705 if opts.dry_run {
706 report.claimed += 1;
707 continue;
708 }
709
710 match self.fire(&hash, &chain, &trigger, state, raw.as_deref(), now, opts) {
711 // Lost the claim between deciding it was free and taking it.
712 // Normal with several nodes on one heartbeat, and it must not
713 // be counted as work done.
714 Ok(outcome) if outcome.skipped_locked => report.skipped_locked += 1,
715 Ok(outcome) => {
716 if !outcome.fired_items.is_empty() {
717 if let Err(e) =
718 self.record_for_composites(&hash, &outcome.fired_items, now, &mut chains)
719 {
720 report.errors.push(e.to_string());
721 }
722 }
723 report.claimed += 1;
724 report.items += outcome.items;
725 report.duplicates += outcome.duplicates;
726 report.runs_started += outcome.runs_started;
727 report.ingested += outcome.ingested;
728 report.unidentifiable += outcome.unidentifiable;
729 for why in &outcome.failures {
730 report.errors.push(format!("{hash}: {why}"));
731 }
732 // #129 — any start failure in this firing means the
733 // cursor was deliberately left unmoved (see `fire`'s
734 // release logic); surfaced apart from `errors` so an
735 // operator sees WHY the same items come back next tick
736 // rather than inferring it.
737 if !outcome.failures.is_empty() {
738 report.cursor_held.push(hash.clone());
739 }
740 if outcome.draining {
741 report.draining.push(hash.clone());
742 }
743 }
744 Err(e) => {
745 // One broken trigger must not stop the others.
746 report.errors.push(e.to_string());
747 }
748 }
749 }
750 Ok(report)
751 }
752
753 // `chain` (#128) is one more independent knob on an already knob-heavy
754 // internal call, in the same spirit as the other `too_many_arguments`
755 // allows in this codebase (areev-store's tuned-recall internals, the
756 // bindings' flat FFI surfaces) — splitting it into a struct would not
757 // make any single call site more readable.
758 #[allow(clippy::too_many_arguments)]
759 fn fire(
760 &self,
761 hash: &str,
762 chain: &Chain,
763 trigger: &Trigger,
764 state: TriggerState,
765 raw: Option<&str>,
766 now: i64,
767 opts: &EvalOptions,
768 ) -> Result<FireOutcome> {
769 // #128 — the claim/release CAS below is keyed on the chain ROOT, not
770 // `hash`. This is the hazard the fix has to get right: `fire` claims
771 // with one key and releases with the same key, so if the two ever
772 // disagreed about which key that is, the release's compare-and-swap
773 // would find no row, `released_ok` would be false, and every firing
774 // of a superseded trigger would permanently report `ClaimLost` — a
775 // trigger that could never usefully claim again. `root` is resolved
776 // exactly once here and reused for every claim/release call in this
777 // function.
778 let root = chain.root();
779
780 // Claim. Losing means another evaluator got here first.
781 let mut claimed = state.clone();
782 claimed.fence = claimed.fence.wrapping_add(1);
783 claimed.claimed_by = Some(opts.node.clone());
784 claimed.lease_until = Some(now + opts.lease.as_millis() as i64);
785 let won = self
786 .facade
787 .with_store(|m| m.put_trigger_state(root, raw, &claimed))
788 .map_err(|e| TriggerError::Storage { detail: e.to_string() })?;
789 if !won {
790 return Ok(FireOutcome { skipped_locked: true, ..Default::default() });
791 }
792 // What a later release must present as `expected`.
793 let claimed_raw = self
794 .facade
795 .with_store(|m| m.trigger_state(root))
796 .map_err(|e| TriggerError::Storage { detail: e.to_string() })?
797 .map(|(_, r)| r);
798
799 let result = self.collect_and_start(hash, chain, trigger, &claimed, now, opts);
800
801 // Release under the fence, whatever happened.
802 let mut released = claimed.clone();
803 released.claimed_by = None;
804 released.lease_until = None;
805 match &result {
806 // Full success: every item that resolved an identity started (or
807 // was ingested, or was recognized as a duplicate) cleanly.
808 Ok(outcome) if outcome.failures.is_empty() => {
809 released.last_fired_at = Some(now);
810 released.consecutive_failures = 0;
811 released.last_error = None;
812 if let Some(c) = &outcome.cursor {
813 released.cursor = Some(c.clone());
814 }
815 if let Some(c) = outcome.op_cursor {
816 released.op_cursor = Some(c);
817 }
818 // A satisfied gate consumes its correlation key. Leaving it
819 // would re-fire the same set on every pass; Flink calls this
820 // the after-match skip strategy, and the conservative choice —
821 // clear only the key that fired — is the one that cannot
822 // swallow an unrelated in-flight match.
823 for k in &outcome.settled_keys {
824 released.partials.remove(k);
825 }
826 prune_partials(&mut released, trigger, now);
827 released.exhausted = schedule::exhausts_after(trigger, now);
828 released.next_due_at = if outcome.draining {
829 // A backlog drains at once rather than waiting out the
830 // interval — a cold start should not take a day.
831 Some(now)
832 } else {
833 schedule::advance_after_firing(trigger, now, now)?
834 };
835 }
836 // #129 — at least one item failed to start a run: the Event add
837 // failed, declared context was unavailable, or the run itself
838 // refused (RUN-E018 and friends). All three mean the item was
839 // NOT processed, so this firing must not report itself clean —
840 // that is the opposite posture from the Err arm below, which
841 // this deliberately mirrors:
842 // - the cursor / op_cursor are left at `claimed`'s values
843 // (never assigned from `outcome`), so the next pass re-asks
844 // the connector for the same page instead of skipping past
845 // unprocessed work;
846 // - `exhausted` is left at `claimed.exhausted` (false, since a
847 // due trigger cannot already be exhausted) rather than
848 // recomputed — a `once` trigger that fails must be allowed
849 // to retry, not be marked as having had its one shot;
850 // - `settled_keys` are NOT removed from `partials` — a
851 // satisfied composite gate whose run refused must stay
852 // satisfied so it retries, not lose its correlation.
853 // Re-polling the same page is safe BECAUSE run ids are keyed on
854 // the chain root (#128): whatever already started is protected
855 // by `start_run`'s dedup fence and comes back as `Duplicate`,
856 // never a second run. What DID happen is still real —
857 // `runs_started`/`ingested`/`duplicates` in the outcome are not
858 // discarded, only the durable cursor advance is withheld.
859 Ok(outcome) => {
860 released.last_fired_at = Some(now);
861 released.consecutive_failures = claimed.consecutive_failures.saturating_add(1);
862 released.last_error = Some(summarize_failures(&outcome.failures));
863 released.next_due_at =
864 Some(schedule::backoff_until(trigger, released.consecutive_failures, now));
865 }
866 Err(e) => {
867 released.consecutive_failures = claimed.consecutive_failures.saturating_add(1);
868 released.last_error = Some(e.to_string());
869 released.next_due_at =
870 Some(schedule::backoff_until(trigger, released.consecutive_failures, now));
871 }
872 }
873 let released_ok = self
874 .facade
875 .with_store(|m| m.put_trigger_state(root, claimed_raw.as_deref(), &released))
876 .map_err(|e| TriggerError::Storage { detail: e.to_string() })?;
877 if !released_ok {
878 // Our lease expired mid-firing and someone else took over. Their
879 // state stands; ours is refused. The work we did is not lost — the
880 // runs we started are journaled and idempotent — but we must not
881 // write back over the new owner.
882 self.journal(hash, trigger, &FireOutcome::default(), Some("claim lost"), now)?;
883 return Err(TriggerError::ClaimLost { trigger: hash.to_string() });
884 }
885
886 let outcome = result?;
887 self.journal(hash, trigger, &outcome, None, now)?;
888 Ok(outcome)
889 }
890
891 /// Get the items for this firing and start a run per item.
892 fn collect_and_start(
893 &self,
894 hash: &str,
895 chain: &Chain,
896 trigger: &Trigger,
897 state: &TriggerState,
898 now: i64,
899 opts: &EvalOptions,
900 ) -> Result<FireOutcome> {
901 let mut outcome = FireOutcome::default();
902
903 let items: Vec<PollItem> = match trigger.kind {
904 TriggerKind::Polling => {
905 let response = self.poll(hash, trigger, state, opts)?;
906 outcome.cursor = response.cursor.clone();
907 outcome.draining = response.more;
908 // First contact seeds the cursor and fires nothing. Otherwise
909 // declaring a mailbox trigger starts a run for every message in
910 // history — Zapier's priming poll, and for the same reason.
911 if state.cursor.is_none() {
912 outcome.seeded = true;
913 return Ok(outcome);
914 }
915 response.items
916 }
917 // A clocked trigger with no external source fires once per
918 // occurrence, carrying the occurrence itself as the item.
919 TriggerKind::Interval | TriggerKind::Schedule | TriggerKind::Once => {
920 vec![PollItem {
921 id: now.to_string(),
922 payload: serde_json::json!({ "at_ms": now }),
923 blobs: Vec::new(),
924 }]
925 }
926 TriggerKind::Memory => {
927 // Recorded so the journal says "seeded" rather than leaving a
928 // firing that produced nothing looking like a quiet memory.
929 outcome.seeded = state.op_cursor.is_none();
930 let (found, cursor) = self.scan_memory(hash, trigger, state, opts)?;
931 outcome.op_cursor = Some(cursor);
932 found
933 }
934 TriggerKind::Composite => {
935 let keys = self.satisfied_keys(hash, chain, trigger, now)?;
936 outcome.settled_keys = keys.clone();
937 keys.into_iter()
938 .map(|k| PollItem {
939 // The correlation value IS the identity: one firing per
940 // correlated set, however many members contributed.
941 id: k.clone(),
942 payload: serde_json::json!({ "correlation": k }),
943 blobs: Vec::new(),
944 })
945 .collect()
946 }
947 // These do not poll. The host owns the listener and hands the
948 // payload to `deliver`, which fires them on the same idempotency
949 // terms; an evaluation pass simply has nothing to do for one.
950 TriggerKind::Webhook | TriggerKind::Manual => Vec::new(),
951 };
952
953 // #93 — store connector blobs in the CAS and rewrite `"blob": "@N"`
954 // payload references to their addresses BEFORE identity resolution,
955 // so a `--dedup-key` pointer into a rewritten field sees the stable
956 // content address rather than a positional index. A contract
957 // violation fails the whole poll here (TRG-E011): fire() then leaves
958 // the cursor unmoved, so nothing is lost — the connector gets fixed
959 // and the same page is re-polled.
960 let mut items = items;
961 let item_refs = self.ingest_item_blobs(hash, &mut items, opts)?;
962
963 outcome.items = items.len();
964 for (item, refs) in items.into_iter().zip(item_refs) {
965 let Some(value) = dedup_value(&item, &trigger.dedup_key) else {
966 outcome.unidentifiable += 1;
967 continue;
968 };
969 outcome.fired_items.push(item.payload.clone());
970 // Keyed on the chain ROOT (#128): the same item seen again after
971 // this trigger has been re-pointed by `SUPERSEDE` still mints
972 // the id it always would have, so `start_run`'s dedup fence
973 // still recognizes it. `legacy_run_ids` is the compatibility
974 // path for a trigger already superseded BEFORE this fix — the
975 // id(s) it would have minted under an older head — and is empty
976 // (so free) for the common never-superseded case.
977 let root = chain.root();
978 let run_id = run_id_for(root, trigger.connector.as_deref(), &value);
979 let legacy_run_ids: Vec<String> = chain
980 .legacy()
981 .iter()
982 .map(|h| run_id_for(h, trigger.connector.as_deref(), &value))
983 .collect();
984 match self.start_run(trigger, &run_id, &legacy_run_ids, &item, hash, &refs) {
985 StartOutcome::Started => outcome.runs_started += 1,
986 StartOutcome::Ingested => outcome.ingested += 1,
987 StartOutcome::Duplicate => outcome.duplicates += 1,
988 StartOutcome::Failed(why) => outcome.failures.push(why),
989 }
990 }
991 Ok(outcome)
992 }
993
994 /// Note a member firing against every composite that declares it.
995 ///
996 /// The correlation value comes from the composite's own `correlate`
997 /// pointer applied to the member's item — so two composites may correlate
998 /// the same member on different fields, which is why this is resolved per
999 /// composite rather than once per member.
1000 fn record_for_composites(
1001 &self,
1002 member: &str,
1003 items: &[serde_json::Value],
1004 now: i64,
1005 chains: &mut HashMap<String, Chain>,
1006 ) -> Result<()> {
1007 for (chash, composite) in self.declarations()? {
1008 let Some(alias) = composite.member_alias(member).map(str::to_string) else {
1009 continue;
1010 };
1011 if composite.kind != TriggerKind::Composite {
1012 continue;
1013 }
1014 // #128 — the composite's OWN chain, not the member's: state
1015 // lives under the composite's root regardless of which member
1016 // just fired. Shares the pass-wide cache, so a composite with
1017 // several members touched this pass walks its chain once.
1018 let chain = self.resolve_chain(&chash, chains)?;
1019 let (mut state, raw) = self.resolve_state(&chain, true)?;
1020
1021 for item in items {
1022 // No correlate pointer means one global match — every member
1023 // firing counts toward the same gate.
1024 let key = match &composite.correlate {
1025 Some(ptr) => match item.pointer(ptr) {
1026 Some(serde_json::Value::String(s)) => s.clone(),
1027 Some(other) => other.to_string(),
1028 // An item that cannot be correlated cannot join a
1029 // correlated gate. Dropping it is right: attributing it
1030 // to an arbitrary key would pair unrelated work.
1031 None => continue,
1032 },
1033 None => String::new(),
1034 };
1035 let entry = state.partials.entry(key).or_insert_with(|| {
1036 areev_store::PartialMatch { members: Default::default(), started_ms: now }
1037 });
1038 // Recorded under the ALIAS, because that is what the gate
1039 // expression names.
1040 entry.members.insert(alias.clone(), now);
1041 }
1042 prune_partials(&mut state, &composite, now);
1043 self.facade
1044 .with_store(|m| m.put_trigger_state(chain.root(), raw.as_deref(), &state))
1045 .map_err(|e| TriggerError::Storage { detail: e.to_string() })?;
1046 }
1047 Ok(())
1048 }
1049
1050 /// Which correlation keys currently satisfy this composite's gate.
1051 fn satisfied_keys(
1052 &self,
1053 hash: &str,
1054 chain: &Chain,
1055 trigger: &Trigger,
1056 now: i64,
1057 ) -> Result<Vec<String>> {
1058 let predicate = trigger
1059 .predicate
1060 .as_ref()
1061 .ok_or_else(|| TriggerError::Malformed {
1062 what: format!("composite trigger {hash} has no predicate"),
1063 })
1064 .and_then(crate::predicate::from_value)?;
1065
1066 // Refuse a gate that names a member the declaration does not carry: it
1067 // could never be satisfied, so the trigger would be silently dead.
1068 for referenced in crate::predicate::referenced_members(&predicate) {
1069 if !trigger.members.contains_key(&referenced) {
1070 return Err(TriggerError::UnknownMember {
1071 trigger: hash.to_string(),
1072 member: referenced,
1073 });
1074 }
1075 }
1076
1077 // #128 — this composite's own partials live under its chain root.
1078 // No adoption dance needed here: by the time `fire` reaches
1079 // `collect_and_start` for this trigger, `run()`'s loop has already
1080 // resolved (and, if needed, adopted) this exact root's state row.
1081 let (state, _) = self
1082 .facade
1083 .with_store(|m| m.trigger_state(chain.root()))
1084 .map_err(|e| TriggerError::Storage { detail: e.to_string() })?
1085 .unwrap_or_default();
1086
1087 let mut out = Vec::new();
1088 for (key, partial) in &state.partials {
1089 if let Some(w) = trigger.window_ms {
1090 if now - partial.started_ms > w {
1091 continue;
1092 }
1093 }
1094 let fired: Vec<String> = partial.members.keys().cloned().collect();
1095 if crate::predicate::gate_satisfied(&predicate, &fired) {
1096 out.push(key.clone());
1097 }
1098 }
1099 Ok(out)
1100 }
1101
1102 /// Grains matching the predicate that appeared since the stored op cursor.
1103 ///
1104 /// Built on the op-log rather than a scan: `changes_since` is an indexed
1105 /// primary-key range read with a monotonic exclusive cursor, so cost is
1106 /// proportional to what changed rather than to how much the memory holds.
1107 ///
1108 /// Considered and rejected: `PRAGMA data_version` as a cheap "did anything
1109 /// change" pre-check. It is documented as unchanged for commits made on the
1110 /// same connection — and Areev is single-writer with one connection, so our
1111 /// own writes are exactly what it cannot see.
1112 fn scan_memory(
1113 &self,
1114 hash: &str,
1115 trigger: &Trigger,
1116 state: &TriggerState,
1117 opts: &EvalOptions,
1118 ) -> Result<(Vec<PollItem>, i64)> {
1119 let predicate = trigger
1120 .predicate
1121 .as_ref()
1122 .ok_or_else(|| TriggerError::Malformed {
1123 what: format!("memory trigger {hash} has no predicate"),
1124 })
1125 .and_then(crate::predicate::from_value)?;
1126
1127 // First contact seeds at the head of the log and matches nothing.
1128 // Otherwise declaring a memory trigger on an established memory fires
1129 // once for every historical grain that matches — the same reason a
1130 // polling trigger primes rather than replaying a mailbox.
1131 let Some(after) = state.op_cursor else {
1132 let head = self
1133 .facade
1134 .with_store(|m| m.head_op_seq())
1135 .map_err(|e| TriggerError::Storage { detail: e.to_string() })?;
1136 return Ok((Vec::new(), head));
1137 };
1138 let ops = self
1139 .facade
1140 .with_store(|m| m.changes_since(after, opts.max_items))
1141 .map_err(|e| TriggerError::Storage { detail: e.to_string() })?;
1142
1143 let mut items = Vec::new();
1144 let mut cursor = after;
1145 for op in ops {
1146 cursor = cursor.max(op.op_seq);
1147 // Additions and supersessions both mean "this is true now".
1148 // A tombstone means the grain is gone, so there is nothing to match.
1149 if op.op == areev_store::OP_FORGET {
1150 continue;
1151 }
1152 let Ok(grain) = self.facade.with_store(|m| m.get(&op.hash)) else {
1153 // A grain that vanished between the op-log read and the fetch
1154 // (erasure, retention) is not an error — it simply no longer
1155 // matches anything.
1156 continue;
1157 };
1158 if !crate::predicate::grain_matches(&grain, &predicate) {
1159 continue;
1160 }
1161 let hex = op.hash.to_hex();
1162 items.push(PollItem {
1163 // The content address IS the identity: the same grain can never
1164 // fire twice, and two evaluators seeing it agree without
1165 // coordinating.
1166 id: hex.clone(),
1167 payload: serde_json::json!({
1168 "hash": hex,
1169 "grain_type": grain.grain_type.as_str(),
1170 "op_seq": op.op_seq,
1171 }),
1172 blobs: Vec::new(),
1173 });
1174 }
1175 Ok((items, cursor))
1176 }
1177
1178 /// Store an item's connector blobs in the CAS and rewrite payload
1179 /// references (#93). Returns one `ContentRef` list per item, parallel
1180 /// to `items`. Any contract violation — undecodable base64, a budget
1181 /// overrun, a dangling `"@N"` — refuses the WHOLE poll with TRG-E011,
1182 /// so the cursor stays put and nothing is silently dropped. `put_blob`
1183 /// is idempotent on content, so a re-polled page costs only the
1184 /// transfer.
1185 fn ingest_item_blobs(
1186 &self,
1187 hash: &str,
1188 items: &mut [PollItem],
1189 opts: &EvalOptions,
1190 ) -> Result<Vec<Vec<ContentRef>>> {
1191 let violation = |detail: String| TriggerError::BlobContract {
1192 trigger: short(hash, 12).to_string(),
1193 detail,
1194 };
1195 let mut response_total = 0usize;
1196 let mut all_refs = Vec::with_capacity(items.len());
1197 for (idx, item) in items.iter_mut().enumerate() {
1198 if item.blobs.is_empty() {
1199 all_refs.push(Vec::new());
1200 continue;
1201 }
1202 let mut item_total = 0usize;
1203 let mut refs = Vec::with_capacity(item.blobs.len());
1204 let mut uris = Vec::with_capacity(item.blobs.len());
1205 for (bi, blob) in item.blobs.iter().enumerate() {
1206 let name = blob.filename.as_deref().unwrap_or("unnamed");
1207 let bytes = areev_core::b64::decode(&blob.b64).ok_or_else(|| {
1208 violation(format!("item {idx} blob {bi} ({name}) is not valid base64"))
1209 })?;
1210 item_total += bytes.len();
1211 response_total += bytes.len();
1212 if item_total > opts.max_blob_bytes_per_item {
1213 return Err(violation(format!(
1214 "item {idx} blobs exceed the per-item budget ({} bytes decoded)",
1215 opts.max_blob_bytes_per_item
1216 )));
1217 }
1218 if response_total > opts.max_blob_bytes_per_response {
1219 return Err(violation(format!(
1220 "response blobs exceed the per-response budget ({} bytes decoded)",
1221 opts.max_blob_bytes_per_response
1222 )));
1223 }
1224 let uri = self
1225 .facade
1226 .with_store(|m| m.put_blob(&bytes))
1227 .map_err(|e| TriggerError::Storage { detail: e.to_string() })?;
1228 refs.push(ContentRef {
1229 uri: uri.clone(),
1230 modality: None,
1231 mime_type: blob.mime.clone(),
1232 size_bytes: Some(bytes.len() as u64),
1233 checksum: Some(uri.trim_start_matches("cas://").to_string()),
1234 metadata: blob.filename.as_ref().map(|f| serde_json::json!({ "filename": f })),
1235 });
1236 uris.push(uri);
1237 }
1238 rewrite_blob_refs(&mut item.payload, &uris)
1239 .map_err(|d| violation(format!("item {idx}: {d}")))?;
1240 // The bytes are in the CAS now; drop the base64 so nothing
1241 // downstream can re-inline it into the Event or the run input.
1242 item.blobs.clear();
1243 all_refs.push(refs);
1244 }
1245 Ok(all_refs)
1246 }
1247
1248 /// Resolve a trigger's `connector_tool` into something executable, or
1249 /// `None` when the trigger names no connector grain and the host command
1250 /// polls as before.
1251 ///
1252 /// Every refusal here happens before the poll has spent anything, and each
1253 /// one names the flag that fixes it — a heartbeat is the surface where a
1254 /// vague failure costs the most, because nobody is reading it.
1255 fn prepare_connector(
1256 &self,
1257 hash: &str,
1258 trigger: &Trigger,
1259 connector_name: &str,
1260 ) -> Result<Option<PreparedConnector>> {
1261 let Some(reference) = trigger
1262 .connector_tool
1263 .as_deref()
1264 .map(str::trim)
1265 .filter(|r| !r.is_empty())
1266 else {
1267 return Ok(None);
1268 };
1269 let refuse = |detail: String| TriggerError::ConnectorCode {
1270 trigger: hash.to_string(),
1271 detail,
1272 };
1273 let Some(cfg) = self.connector_code.as_ref() else {
1274 return Err(refuse(format!(
1275 "connector {connector_name:?} is a grain ({reference}) but this host runs no \
1276 code-carrying connectors — pass --allow-executor <address> (and --sandbox-cmd \
1277 for a wasm module)"
1278 )));
1279 };
1280 let hex = areev_core::types::strip_grain_scheme(reference);
1281 let h = Hash::from_hex(hex).map_err(|e| {
1282 refuse(format!("connector_tool {reference:?} is not a content address: {e}"))
1283 })?;
1284 let g = self
1285 .facade
1286 .with_store(|m| m.get(&h))
1287 .map_err(|e| refuse(format!("connector Definition {hex} is unreadable: {e}")))?;
1288 if g.grain_type != GrainType::Tool {
1289 return Err(refuse(format!(
1290 "connector_tool {hex} is a {:?} grain, not a Tool Definition",
1291 g.grain_type
1292 )));
1293 }
1294 // ONE reader of a Definition's executable shape, shared with
1295 // `RunManifest::resolve`: a connector that ran under different rules
1296 // than a node binding the same grain would be a second implementation
1297 // of the pin, the runtime table and the capability check — and the one
1298 // that drifts is this one, because nobody is watching a poll.
1299 let pin = areev_run::pin_from_definition(connector_name, &h, &g)
1300 .map_err(|e| refuse(e.to_string()))?;
1301 let Some(uri) = pin.executor_uri.clone() else {
1302 return Err(refuse(format!(
1303 "connector Definition {hex} names no executor_uri — a connector grain IS its \
1304 code, so there is nothing to run"
1305 )));
1306 };
1307 // The host pin, checked here so the refusal can name the address to
1308 // copy. `execute_code` checks it again on the way in; that one is the
1309 // guarantee, this one is the error message.
1310 let addr = normalize_address(&uri);
1311 if !cfg.allow.iter().any(|a| normalize_address(a) == addr) {
1312 return Err(refuse(format!(
1313 "connector {connector_name:?} names {uri}, which this host did not pin — \
1314 add --allow-executor {addr}"
1315 )));
1316 }
1317 if pin.runtime.as_deref().is_some_and(areev_run::is_sandbox_runtime)
1318 && cfg.sandbox_cmd.is_none()
1319 {
1320 return Err(refuse(format!(
1321 "connector {connector_name:?} declares runtime {:?}, which needs the sandbox — \
1322 pass --sandbox-cmd areev-sandbox",
1323 pin.runtime.as_deref().unwrap_or_default()
1324 )));
1325 }
1326 // `get_blob` verifies the digest on every read, so these bytes ARE the
1327 // address the trigger named.
1328 let bytes = self
1329 .facade
1330 .with_store(|m| m.get_blob(&uri))
1331 .map_err(|e| refuse(format!("connector blob {uri} is unreadable: {e}")))?;
1332 let wants_blobs = pin
1333 .capabilities
1334 .as_ref()
1335 .and_then(|v| areev_run::Declaration::parse(v).ok())
1336 .is_some_and(|d| d.declares_blob_read());
1337 let config = g.fields.get("config").filter(|v| !v.is_null()).cloned();
1338 Ok(Some(PreparedConnector {
1339 tool_hash: h.to_hex(),
1340 code: areev_run::PreparedCode {
1341 uri,
1342 bytes,
1343 runtime: pin.runtime,
1344 limits: pin.runtime_limits,
1345 capabilities: pin.capabilities,
1346 },
1347 wants_blobs,
1348 config,
1349 }))
1350 }
1351
1352 /// The executor a grain connector runs under, bound to THIS poll's broker.
1353 ///
1354 /// Built per poll rather than once on the evaluator because the broker is
1355 /// per poll: it is started with the trigger's own allowlist and stops when
1356 /// the call returns, and a handle to a broker that has stopped would hand
1357 /// the module a dead token.
1358 fn code_executor(&self, broker: &Arc<areev_run::Broker>) -> areev_run::CodeExecutor {
1359 let cfg = self.connector_code.clone().unwrap_or_default();
1360 let mut ce = areev_run::CodeExecutor::new(Arc::new(NoConnectorFallback));
1361 for addr in &cfg.allow {
1362 ce = ce.allow(addr);
1363 }
1364 if let Some(dir) = cfg.cache_dir {
1365 ce = ce.cache_dir(dir);
1366 }
1367 if let Some(cmd) = cfg.sandbox_cmd.as_deref() {
1368 ce = ce.sandbox_cmd(cmd);
1369 }
1370 if let Some(t) = cfg.timeout {
1371 ce = ce.with_timeout(t);
1372 }
1373 if let Some(env) = cfg.env {
1374 ce = ce.with_env_policy(env);
1375 }
1376 ce.with_egress(areev_run::EgressHandle::new(Arc::clone(broker)))
1377 }
1378
1379 fn poll(
1380 &self,
1381 hash: &str,
1382 trigger: &Trigger,
1383 state: &TriggerState,
1384 opts: &EvalOptions,
1385 ) -> Result<PollResponse> {
1386 let connector_name = trigger.connector.as_deref().unwrap_or_default();
1387 // A connector named by content address (#185) is resolved, pinned and
1388 // read BEFORE a broker starts: a Definition this host did not pin must
1389 // refuse having spent nothing — no port, no token, no upstream call.
1390 let prepared = self.prepare_connector(hash, trigger, connector_name)?;
1391 if prepared.is_none() && self.connector.is_none() {
1392 return Err(TriggerError::NoConnector {
1393 trigger: hash.to_string(),
1394 connector: connector_name.to_string(),
1395 });
1396 }
1397 // Start a broker for this call, scoped to this trigger's allowlist. The
1398 // connector is handed its URL — never a token — so a compromised
1399 // connector has nothing to exfiltrate and nowhere undeclared to send
1400 // it. The broker stops when this call returns.
1401 let policy = areev_run::EgressPolicy::from_config(trigger.config.as_ref())
1402 .map_err(|what| TriggerError::Malformed { what })?;
1403 // One connector runs per pass, so there is no second caller to tell it
1404 // apart from: a default grant covers it. Its methods are whatever the
1405 // connector needs to poll, and the credentials are the ones this host
1406 // configured — naming one it was not given still fails.
1407 let grant = self.credentials.keys().fold(
1408 areev_run::CallerGrant::new()
1409 .method("GET")
1410 .method("POST")
1411 .method("PUT")
1412 .method("PATCH")
1413 .method("DELETE"),
1414 |g, name| g.credential(name),
1415 );
1416 // The same grant twice, under two names, because the broker
1417 // identifies a caller by the TOKEN it presents: the default token maps
1418 // to no caller at all, and a capability declaration is registered
1419 // under a name (#185). A grain connector declares
1420 // `{"blob": {"read": true}}` as tool `<connector>`, so it needs a
1421 // token that resolves back to `<connector>` — otherwise its own
1422 // declaration is invisible to the check that reads it, and every blob
1423 // read is refused as undeclared. The default stays for the subprocess
1424 // connector, which presents a token and names nothing.
1425 let grants = areev_run::EgressGrants::new()
1426 .default_for_all(grant.clone())
1427 .grant(connector_name, grant);
1428 let broker = Arc::new(
1429 areev_run::Broker::start(policy, self.credentials.clone(), grants, "TRG-E009")
1430 .map_err(|detail| TriggerError::Storage { detail })?,
1431 );
1432 // A connector module that declared `{"blob": {"read": true}}` reads
1433 // the memory's stored bytes through this same broker on the same
1434 // token (#106) — the attachment an earlier poll filed, by address.
1435 // The read never opens the memory, which is what makes it safe while
1436 // the evaluator holds it.
1437 if prepared.as_ref().is_some_and(|p| p.wants_blobs) {
1438 let locator = self
1439 .connector_code
1440 .as_ref()
1441 .and_then(|c| c.db_locator.as_deref())
1442 .ok_or_else(|| TriggerError::ConnectorCode {
1443 trigger: hash.to_string(),
1444 detail: format!(
1445 "connector {connector_name:?} declares {{\"blob\": {{\"read\": true}}}} \
1446 but this host wired the evaluator no memory locator, so the broker \
1447 cannot serve a blob read"
1448 ),
1449 })?;
1450 broker.serve_blobs(locator);
1451 }
1452
1453 let config = merged_config(
1454 prepared.as_ref().and_then(|p| p.config.as_ref()),
1455 trigger.config.as_ref(),
1456 );
1457 let request = PollRequest {
1458 trigger: hash,
1459 connector: connector_name,
1460 scope: trigger.scope.as_deref(),
1461 cursor: state.cursor.as_deref(),
1462 max_items: opts.max_items,
1463 config: config.as_ref(),
1464 };
1465 let payload = serde_json::to_value(&request).map_err(|e| TriggerError::ConnectorFailed {
1466 trigger: hash.to_string(),
1467 detail: format!("serialize request: {e}"),
1468 })?;
1469 // The idempotency key ties this call to this occurrence, so a connector
1470 // that deduplicates on it sees a retry as a retry.
1471 let idem = format!("{hash}:{}", state.fence);
1472 static EGRESS_ENV: std::sync::Mutex<()> = std::sync::Mutex::new(());
1473 let result = match &prepared {
1474 // The connector is a grain: the pinned blob runs under the same
1475 // `CodeExecutor` the run path uses, so the sandbox argv, the
1476 // capability registration, the per-call broker handshake and the
1477 // cleared environment are ONE implementation rather than a second
1478 // one that drifts. It takes its broker token per call rather than
1479 // through the process environment, so the mutex below is not its
1480 // hazard to hold.
1481 Some(p) => self.code_executor(&broker).execute_code(
1482 connector_name,
1483 &p.tool_hash,
1484 &p.code,
1485 &payload,
1486 &idem,
1487 ),
1488 // The connector reads AREEV_EGRESS_URL out of its environment. The
1489 // spawn seam scrubs whatever `--passphrase-env` and `--token-env`
1490 // name, so this is the only network affordance it has from us.
1491 //
1492 // The variable is PROCESS-GLOBAL, and `HostToolExecutor::execute`
1493 // has no per-call environment, so two evaluators in one process
1494 // would race: one could hand its broker's URL to the other's
1495 // connector, pointing it at the wrong allowlist. The lock makes
1496 // the set→spawn→unset window exclusive. Production runs one
1497 // evaluator per process, so this serialises nothing that was
1498 // parallel; it closes a hazard that only exists in-process (and
1499 // that our own threaded tests could reach).
1500 None => {
1501 let exec = self.connector.as_ref().expect("checked above");
1502 let _guard = EGRESS_ENV.lock().unwrap_or_else(|e| e.into_inner());
1503 std::env::set_var("AREEV_EGRESS_URL", broker.url());
1504 // The broker authenticates its callers: loopback is not an
1505 // authorization, and without the token every call would 401.
1506 if let Some(t) = broker.token_for(connector_name) {
1507 std::env::set_var("AREEV_EGRESS_TOKEN", t);
1508 }
1509 let r = exec.execute(connector_name, hash, &payload, &idem);
1510 std::env::remove_var("AREEV_EGRESS_URL");
1511 std::env::remove_var("AREEV_EGRESS_TOKEN");
1512 r
1513 }
1514 };
1515
1516 // Surface a refused destination as its own error rather than letting it
1517 // read as a generic connector failure — a connector reaching somewhere
1518 // undeclared is a different problem from one that crashed.
1519 let refused = broker.refusals();
1520 if !refused.is_empty() {
1521 return Err(TriggerError::EgressRefused {
1522 trigger: hash.to_string(),
1523 host: refused
1524 .iter()
1525 .map(|r| format!("{} ({})", r.destination, r.reason))
1526 .collect::<Vec<_>>()
1527 .join(", "),
1528 });
1529 }
1530 match result {
1531 areev_run::ExecResult::Ok(v) => {
1532 // `{"error": …}` is the shape every blessed blob fails in, and
1533 // `PollResponse` would deserialize it into an empty page with
1534 // no cursor — a source that is down, reported as a source with
1535 // nothing new, on every tick, silently. A poll that answered an
1536 // error FAILED: the claim is released, the cursor stays, and
1537 // the backoff applies (#231).
1538 if let Some(detail) = v
1539 .get("error")
1540 .and_then(|e| e.as_str())
1541 .map(str::trim)
1542 .filter(|e| !e.is_empty())
1543 {
1544 let code = v.get("code").and_then(|c| c.as_str()).unwrap_or_default();
1545 let named =
1546 if code.is_empty() { String::new() } else { format!(" [{code}]") };
1547 return Err(TriggerError::ConnectorFailed {
1548 trigger: hash.to_string(),
1549 detail: format!("the connector reported an error{named}: {detail}"),
1550 });
1551 }
1552 serde_json::from_value(v).map_err(|e| TriggerError::ConnectorFailed {
1553 trigger: hash.to_string(),
1554 detail: format!("response is not a poll response: {e}"),
1555 })
1556 }
1557 areev_run::ExecResult::Err { detail, .. } => {
1558 Err(TriggerError::ConnectorFailed { trigger: hash.to_string(), detail })
1559 }
1560 }
1561 }
1562
1563 /// Record the item as an Event and start the bound workflow.
1564 ///
1565 /// `legacy_run_ids` (#128) covers a trigger that was superseded BEFORE
1566 /// this fix shipped, when the run id was minted from the (then-current)
1567 /// head rather than the chain root: each entry is the id this same item
1568 /// would have gotten under an older head. Empty for a never-superseded
1569 /// trigger, so it costs nothing in the common case.
1570 fn start_run(
1571 &self,
1572 trigger: &Trigger,
1573 run_id: &str,
1574 legacy_run_ids: &[String],
1575 item: &PollItem,
1576 trigger_hash: &str,
1577 content_refs: &[ContentRef],
1578 ) -> StartOutcome {
1579 // Idempotency has to hold whether or not a runtime is wired in. With a
1580 // starter, the runtime's duplicate-run-id refusal is the guard; without
1581 // one, nothing else would stop the same item being re-ingested on every
1582 // poll forever, because an Event's `created_at` differs per firing and
1583 // so content addressing does not collapse them.
1584 let seen = self
1585 .facade
1586 .with_store(|m| m.run_grains(&self.ns, run_id, 0, 1))
1587 .map(|g| !g.is_empty())
1588 .unwrap_or(false);
1589 if seen {
1590 return StartOutcome::Duplicate;
1591 }
1592 // #128 compat — an item already processed under a pre-fix
1593 // (head-keyed) run id must still be recognized, or a trigger
1594 // superseded before this fix reprocesses its whole backlog the
1595 // first time it is evaluated after upgrading.
1596 for legacy_id in legacy_run_ids {
1597 let seen_legacy = self
1598 .facade
1599 .with_store(|m| m.run_grains(&self.ns, legacy_id, 0, 1))
1600 .map(|g| !g.is_empty())
1601 .unwrap_or(false);
1602 if seen_legacy {
1603 return StartOutcome::Duplicate;
1604 }
1605 }
1606
1607 // The ingest Event is built here but — #129 — WRITTEN only once the
1608 // outcome is one this dedup check should treat as "seen": Ingested
1609 // (no starter, so the Event itself IS the completed unit of work),
1610 // Started, or a starter-reported Duplicate (the runtime already has
1611 // it covered, possibly from elsewhere). A starter-reported `Failed`
1612 // writes NOTHING, so a retry's `seen` check above finds nothing and
1613 // genuinely retries `starter.start()` — before this, the Event was
1614 // always written first, so a failed start still left a `run_id`-
1615 // bearing grain behind that made every future retry of the SAME
1616 // item look like an already-processed duplicate, permanently
1617 // stranding it despite #129's cursor hold.
1618 let build_event = || {
1619 let mut event = Event::new(&item.payload.to_string())
1620 .namespace(&self.ns)
1621 .extra_field("trigger", serde_json::json!(trigger_hash))
1622 .extra_field("run_id", serde_json::json!(run_id));
1623 if let Some(c) = &trigger.connector {
1624 event = event.extra_field("connector", serde_json::json!(c));
1625 }
1626 // #93 — the item's stored blobs, referenced the same way a
1627 // host-driven ingest references them: through `content_refs`,
1628 // which is what keeps them alive through GC, carries them in
1629 // bundles, and lets erasure's sole-reference reclamation find
1630 // them.
1631 for cr in content_refs {
1632 event = event.content_ref(cr.clone());
1633 }
1634 event
1635 };
1636
1637 let Some(starter) = &self.starter else {
1638 // Ingest-only: the item is recorded, nothing is executed. Reported
1639 // as ingested rather than as a run, because claiming a run started
1640 // when none did would make the report lie.
1641 return match self.facade.with_store(|m| m.add(&build_event())) {
1642 Ok(_) => StartOutcome::Ingested,
1643 Err(e) => StartOutcome::Failed(format!("ingest: {e}")),
1644 };
1645 };
1646 let mut input = serde_json::json!({
1647 "trigger": trigger_hash,
1648 "connector": trigger.connector,
1649 "scope": trigger.scope,
1650 "item": item.payload,
1651 });
1652 // Declared context (#85): the EVALUATOR assembles it, because it is
1653 // the one party that already holds the memory — on the embedded
1654 // backend a tool inside the run cannot open the file its own run
1655 // locks. Fail closed: a trigger that declared context must not fire
1656 // blind, so a missing query or a failed read refuses the firing (the
1657 // evaluator's normal retry/backoff applies) rather than starting a
1658 // run without the context its declaration promised.
1659 if let Some(spec) = &trigger.context_query {
1660 match self.assemble_context(spec, &item.payload) {
1661 Ok(ctx) => {
1662 input["context"] = ctx;
1663 }
1664 Err(why) => {
1665 return StartOutcome::Failed(format!("context_query \"{spec}\": {why}"));
1666 }
1667 }
1668 }
1669 // Read through whatever scheme prefix the declaration spelled the
1670 // reference with (#73): `sha256:<hex>` validated, listed, and reported
1671 // `waiting` forever, then died here on `FMT-E001: invalid hex hash`.
1672 match starter.start(trigger.workflow_hash(), run_id, input) {
1673 // The run is durably underway (or already was, started elsewhere)
1674 // before we ever try to write the ingest Event, so a failure to
1675 // write it here does not need to be reported as a failed firing:
1676 // a retry would find `starter.start()` itself now reporting
1677 // `Duplicate` (the runtime's own dedup), never a second run. Best
1678 // effort is enough; the Event is audit/context linkage, not the
1679 // run's own record of itself.
1680 StartResult::Started => {
1681 let _ = self.facade.with_store(|m| m.add(&build_event()));
1682 StartOutcome::Started
1683 }
1684 StartResult::Duplicate => {
1685 let _ = self.facade.with_store(|m| m.add(&build_event()));
1686 StartOutcome::Duplicate
1687 }
1688 StartResult::Failed(why) => StartOutcome::Failed(why),
1689 }
1690 }
1691
1692 /// Run the trigger's saved query (`RUN "name"($p = v, …)`) against the
1693 /// facade this evaluator already holds and return the result payload.
1694 /// The executor re-validates the saved body as read-only at execution,
1695 /// and the destructive cap is off — a context query can never mutate.
1696 ///
1697 /// #92 — the declaration may bind the query's parameters from the
1698 /// firing item (`name($session = /session)`): each JSON pointer is
1699 /// resolved against the item's payload with `--dedup-key`'s machinery
1700 /// and semantics. Fail-closed like the rest of declared context: an
1701 /// unresolvable pointer, or one landing on a non-scalar, refuses the
1702 /// firing rather than running the query with a hole in it. Values ride
1703 /// as a parsed AST (`RunQueryStmt`), never inside CAL text, so an
1704 /// untrusted payload value cannot inject CAL.
1705 fn assemble_context(
1706 &self,
1707 spec: &str,
1708 payload: &serde_json::Value,
1709 ) -> std::result::Result<serde_json::Value, String> {
1710 let spec = crate::context::parse_context_query(spec)?;
1711 let mut bindings: Vec<(String, areev_cal::ast::Value)> = Vec::new();
1712 for (param, pointer) in &spec.bindings {
1713 let v = payload.pointer(pointer).ok_or_else(|| {
1714 format!("parameter ${param}: pointer {pointer} does not resolve against the firing item")
1715 })?;
1716 let value = match v {
1717 serde_json::Value::String(s) => {
1718 areev_cal::ast::Value::String { value: s.clone() }
1719 }
1720 serde_json::Value::Number(n) => areev_cal::ast::Value::Number {
1721 value: n.as_f64().ok_or_else(|| {
1722 format!("parameter ${param}: {pointer} is not a representable number")
1723 })?,
1724 },
1725 serde_json::Value::Bool(b) => areev_cal::ast::Value::Boolean { value: *b },
1726 other => {
1727 return Err(format!(
1728 "parameter ${param}: pointer {pointer} must land on a scalar, found {}",
1729 match other {
1730 serde_json::Value::Null => "null",
1731 serde_json::Value::Array(_) => "an array",
1732 _ => "an object",
1733 }
1734 ))
1735 }
1736 };
1737 bindings.push((param.clone(), value));
1738 }
1739 let ex = areev_cal::CalExecutor::new(areev_cal::CalExecutorConfig {
1740 allow_destructive_ops: false,
1741 ..areev_cal::CalExecutorConfig::default()
1742 });
1743 let query = areev_cal::ast::CalQuery {
1744 version: Default::default(),
1745 statement: areev_cal::ast::CalStatement::RunQuery(areev_cal::ast::RunQueryStmt {
1746 name: spec.name.clone(),
1747 bindings,
1748 span: None,
1749 }),
1750 pipeline: Vec::new(),
1751 with_options: Vec::new(),
1752 format: None,
1753 let_bindings: Vec::new(),
1754 user_vars: Default::default(),
1755 let_values: Default::default(),
1756 warnings: Vec::new(),
1757 };
1758 let res = ex
1759 .execute_parsed(query, &format!("RUN \"{}\"", spec.name), self.facade.as_ref())
1760 .map_err(|e| e.to_string())?;
1761 serde_json::to_value(&res.result).map_err(|e| e.to_string())
1762 }
1763
1764 /// One Observation per firing: what fired, what it produced, what it
1765 /// skipped. An unfired trigger is invisible otherwise.
1766 fn journal(
1767 &self,
1768 hash: &str,
1769 trigger: &Trigger,
1770 outcome: &FireOutcome,
1771 note: Option<&str>,
1772 now: i64,
1773 ) -> Result<()> {
1774 let mut obs = Observation::new("areev:trigger", "trigger:firing")
1775 .namespace(TRIGGER_NS)
1776 .created_at(now)
1777 .extra_field("trigger", serde_json::json!(hash))
1778 .extra_field("kind", serde_json::json!(trigger.kind.as_str()))
1779 .extra_field("workflow", serde_json::json!(trigger.workflow))
1780 .extra_field("items", serde_json::json!(outcome.items))
1781 .extra_field("runs_started", serde_json::json!(outcome.runs_started))
1782 .extra_field("duplicates", serde_json::json!(outcome.duplicates));
1783 // Without these, a firing where every item lacked the dedup key reads
1784 // as "items 5, runs_started 0, duplicates 0" -- and that last zero
1785 // actively misleads, saying the items were not skipped as duplicates
1786 // without saying why they were skipped at all. Emitted only when
1787 // non-zero, so an ordinary firing stays as small as it was.
1788 if outcome.unidentifiable > 0 {
1789 obs = obs.extra_field("unidentifiable", serde_json::json!(outcome.unidentifiable));
1790 }
1791 if outcome.ingested > 0 {
1792 obs = obs.extra_field("ingested", serde_json::json!(outcome.ingested));
1793 }
1794 if !outcome.failures.is_empty() {
1795 obs = obs.extra_field("failures", serde_json::json!(outcome.failures));
1796 // #129 — names WHY the cursor did not move this firing, so the
1797 // audit record does not require the reader to infer it from
1798 // `duplicates` staying flat on the next firing.
1799 obs = obs.extra_field("cursor_held", serde_json::json!(true));
1800 }
1801 if outcome.seeded {
1802 obs = obs.extra_field("seeded", serde_json::json!(true));
1803 }
1804 if let Some(n) = note {
1805 obs = obs.extra_field("note", serde_json::json!(n));
1806 }
1807 self.facade
1808 .with_store(|m| m.add_if_novel(&obs))
1809 .map(|_| ())
1810 .map_err(|e| TriggerError::Storage { detail: e.to_string() })
1811 }
1812
1813 /// The current head of this trigger's PLAN, when the plan has been edited
1814 /// since the trigger was declared. `None` when the trigger already points
1815 /// at the live version — the overwhelmingly common case, and one forward
1816 /// point read to establish.
1817 ///
1818 /// Deliberately a report and not a redirect. Following the chain at fire
1819 /// time would mean a plan edit silently changes what an unattended
1820 /// heartbeat runs, which is exactly what content-addressed binding exists
1821 /// to prevent; the operator moves the pointer with `trigger retarget` when
1822 /// they mean to.
1823 pub fn plan_head_if_stale(&self, t: &Trigger) -> Result<Option<String>> {
1824 let Ok(h) = Hash::from_hex(t.workflow_hash()) else {
1825 // Not a content address at all — a different complaint, and
1826 // `unusable` is where that one is reported.
1827 return Ok(None);
1828 };
1829 let head = self
1830 .facade
1831 .with_store(|m| m.current_head(&h))
1832 .map_err(|e| TriggerError::Storage { detail: e.to_string() })?;
1833 Ok((head != h).then(|| head.to_hex()))
1834 }
1835
1836 /// Resolve (and cache) the supersession chain for one trigger's head
1837 /// hash within one evaluation pass (#128).
1838 ///
1839 /// `cache` lives on the CALLER's stack — a fresh `HashMap` per `run()`,
1840 /// `deliver()`, or `status()` invocation — so a chain is walked AT MOST
1841 /// ONCE per trigger per pass no matter how many items it fires or how
1842 /// many composites settle against it in that pass. Do not add a call
1843 /// site that walks the chain per item; thread the already-resolved
1844 /// [`Chain`] through instead, the way `fire`/`collect_and_start` do.
1845 fn resolve_chain(&self, hash: &str, cache: &mut HashMap<String, Chain>) -> Result<Chain> {
1846 if let Some(c) = cache.get(hash) {
1847 return Ok(c.clone());
1848 }
1849 let h = Hash::from_hex(hash).map_err(|e| TriggerError::Storage { detail: e.to_string() })?;
1850 let hashes: Vec<String> = self
1851 .facade
1852 .with_store(|m| m.supersession_chain(&h))
1853 .map_err(|e| TriggerError::Storage { detail: e.to_string() })?
1854 .into_iter()
1855 .map(|hh| hh.to_hex())
1856 .collect();
1857 let chain = Chain(hashes);
1858 cache.insert(hash.to_string(), chain.clone());
1859 Ok(chain)
1860 }
1861
1862 /// Read a trigger's evaluation state through its resolved chain, with
1863 /// compatibility for a trigger superseded BEFORE #128 shipped (its state
1864 /// was keyed on the head hash of the moment, not the chain root).
1865 ///
1866 /// Always tries the ROOT key first. For a trigger that has never been
1867 /// superseded this is the ONLY path ever taken — `chain.root() ==` the
1868 /// hash this crate always keyed state on — so the common case is a
1869 /// byte-identical lookup to before this feature existed.
1870 ///
1871 /// If the root row is absent, every other hash in the chain
1872 /// (`chain.legacy()`) is checked for a row written under an older head.
1873 /// Several may exist if the trigger was superseded more than once
1874 /// before upgrading; [`state_is_more_advanced`] picks whichever
1875 /// represents the most real progress, so the survivor is never the one
1876 /// that would silently throw away a further-along cursor.
1877 ///
1878 /// `adopt = true` persists the found legacy state under the root and
1879 /// deletes the legacy row — write the new row FIRST, delete the legacy
1880 /// row SECOND, so a crash in between leaves a harmless duplicate (the
1881 /// legacy row is simply never consulted again once the root row exists)
1882 /// rather than losing the cursor. Pass `true` only from a caller that
1883 /// already owns write access for real evaluation work (`run`,
1884 /// `deliver`, composite settlement); `status`/`show` pass `false` so a
1885 /// pure inspection command has no side effect. Adoption is best-effort
1886 /// against a read-only-opened memory: `STO-E004` is swallowed and the
1887 /// found state is still returned, just not yet persisted — the next
1888 /// pass with write access adopts it.
1889 fn resolve_state(&self, chain: &Chain, adopt: bool) -> Result<(TriggerState, Option<String>)> {
1890 let root = chain.root();
1891 if let Some((state, raw)) = self
1892 .facade
1893 .with_store(|m| m.trigger_state(root))
1894 .map_err(|e| TriggerError::Storage { detail: e.to_string() })?
1895 {
1896 return Ok((state, Some(raw)));
1897 }
1898
1899 let mut best: Option<(&str, TriggerState)> = None;
1900 for legacy_hash in chain.legacy() {
1901 if let Some((state, _raw)) = self
1902 .facade
1903 .with_store(|m| m.trigger_state(legacy_hash))
1904 .map_err(|e| TriggerError::Storage { detail: e.to_string() })?
1905 {
1906 let better = best
1907 .as_ref()
1908 .is_none_or(|(_, current)| state_is_more_advanced(&state, current));
1909 if better {
1910 best = Some((legacy_hash.as_str(), state));
1911 }
1912 }
1913 }
1914 let Some((legacy_hash, state)) = best else {
1915 // No row anywhere in the chain: a genuinely new trigger.
1916 return Ok((TriggerState::default(), None));
1917 };
1918 if !adopt {
1919 return Ok((state, None));
1920 }
1921
1922 // `expected=None` only claims an ABSENT row (`INSERT OR IGNORE`), so
1923 // losing a race to another evaluator adopting the same legacy row
1924 // concurrently is a no-op, never a clobber — either way the legacy
1925 // row is redundant now.
1926 match self.facade.with_store(|m| m.put_trigger_state(root, None, &state)) {
1927 Ok(_) => {}
1928 Err(AreevError::ReadOnly(_)) => return Ok((state, None)),
1929 Err(e) => return Err(TriggerError::Storage { detail: e.to_string() }),
1930 }
1931 if let Err(e) = self.facade.with_store(|m| m.clear_trigger_state(legacy_hash)) {
1932 if !matches!(e, AreevError::ReadOnly(_)) {
1933 return Err(TriggerError::Storage { detail: e.to_string() });
1934 }
1935 }
1936 let (state, raw) = self
1937 .facade
1938 .with_store(|m| m.trigger_state(root))
1939 .map_err(|e| TriggerError::Storage { detail: e.to_string() })?
1940 .ok_or_else(|| TriggerError::Storage {
1941 detail: format!(
1942 "adopted trigger state for {root} vanished immediately after writing"
1943 ),
1944 })?;
1945 Ok((state, Some(raw)))
1946 }
1947}
1948
1949/// A trigger's supersession chain for the duration of one evaluation pass:
1950/// every content-address hash in its edit history, HEAD (the declaration's
1951/// current hash — what the caller resolved from) first, ROOT last (#128).
1952///
1953/// Evaluation state (`trg:<hash>`) and derived run ids are keyed on the
1954/// ROOT, never the head: a Trigger grain re-pointed by `SUPERSEDE` mints a
1955/// new head hash for the SAME standing rule (`docs/loop.md`'s
1956/// propose→approve→apply cycle re-points a trigger exactly this way), and
1957/// keying on the head alone would orphan the cursor and the dedup fence
1958/// every time a recommendation is applied. For a trigger that has never
1959/// been superseded, `root() == head`, so every key derived from it is
1960/// byte-identical to what this crate always computed — the no-op guarantee
1961/// the fix has to preserve.
1962#[derive(Debug, Clone)]
1963struct Chain(Vec<String>);
1964
1965impl Chain {
1966 /// The key evaluation state and run ids are actually resolved against.
1967 fn root(&self) -> &str {
1968 self.0.last().map(String::as_str).expect("a chain always has at least its head")
1969 }
1970
1971 /// Every hash besides the root — the identities a pre-#128 evaluator
1972 /// might have keyed state or a run id under. Empty for a
1973 /// never-superseded trigger, so the compatibility paths that walk this
1974 /// cost nothing in the common case.
1975 fn legacy(&self) -> &[String] {
1976 &self.0[..self.0.len() - 1]
1977 }
1978}
1979
1980/// Which of two legacy state rows found along a chain is "further along"
1981/// and should be adopted, when more than one exists (a trigger superseded
1982/// more than once before #128 shipped). `last_fired_at` is the most direct
1983/// evidence of real progress across every trigger kind — the row that
1984/// fired most recently is the row whose cursor best reflects what has
1985/// actually been processed. `op_cursor` (a `memory` trigger's position) and
1986/// then "has this row ever polled at all" break a tie where neither
1987/// candidate has ever fired.
1988fn state_is_more_advanced(candidate: &TriggerState, current_best: &TriggerState) -> bool {
1989 (candidate.last_fired_at, candidate.op_cursor, candidate.cursor.is_some())
1990 > (current_best.last_fired_at, current_best.op_cursor, current_best.cursor.is_some())
1991}
1992
1993/// One line for `released.last_error` (#129): several items in one firing
1994/// can fail for different reasons, and `last_error` is a single string.
1995fn summarize_failures(failures: &[String]) -> String {
1996 match failures {
1997 [one] => one.clone(),
1998 many => format!("{} item(s) failed to start: {}", many.len(), many.join("; ")),
1999 }
2000}
2001
2002/// Drop partial matches past their window.
2003///
2004/// Without this, a composite that never completes accumulates keys forever and
2005/// a stale half-match eventually pairs with unrelated work — the exact failure
2006/// Argo Events' reset cron exists to paper over.
2007fn prune_partials(state: &mut TriggerState, trigger: &Trigger, now: i64) {
2008 let Some(window) = trigger.window_ms else {
2009 return;
2010 };
2011 state.partials.retain(|_, p| now - p.started_ms <= window);
2012}
2013
2014#[derive(Debug, Default, Clone)]
2015struct FireOutcome {
2016 items: usize,
2017 runs_started: usize,
2018 ingested: usize,
2019 duplicates: usize,
2020 unidentifiable: usize,
2021 failures: Vec<String>,
2022 cursor: Option<String>,
2023 /// The payloads this firing produced, so a composite can pull its
2024 /// correlation value out of them.
2025 fired_items: Vec<serde_json::Value>,
2026 /// Correlation keys a composite consumed, cleared on release.
2027 settled_keys: Vec<String>,
2028 op_cursor: Option<i64>,
2029 draining: bool,
2030 seeded: bool,
2031 skipped_locked: bool,
2032}
2033
2034enum StartOutcome {
2035 Started,
2036 /// Recorded but not executed — no runtime is wired in.
2037 Ingested,
2038 Duplicate,
2039 Failed(String),
2040}
2041
2042/// The run id for one firing of one item.
2043///
2044/// Derived from `(trigger, connector, dedup value)`, so the same item seen
2045/// twice — connector replay, overlapping cursors, two nodes racing a lease
2046/// boundary — produces the same id. `areev run start` refuses a duplicate, so
2047/// the second attempt is a recorded skip rather than a second run. At-most-once
2048/// firing with no lease, no fencing token, and no clock assumption.
2049///
2050/// The connector is part of the identity because CloudEvents made that
2051/// normative for the same reason: `source` + `id` is unique, `id` alone is only
2052/// unique within a producer. Two connectors sharing an id space must not
2053/// collide.
2054///
2055/// **`trigger_hash` is the chain ROOT (#128), not necessarily the
2056/// declaration's current head.** A `Trigger` grain re-pointed by
2057/// `SUPERSEDE` mints a new head for the same standing rule, and this
2058/// function has no way to know that on its own — a caller that passed the
2059/// head would mint a fresh id for an item already processed under the old
2060/// head, defeating the very dedup fence this function exists to provide.
2061/// Every call site resolves the root once per pass (`Evaluator::
2062/// resolve_chain`) and passes that; this function's signature and hashing
2063/// are otherwise unchanged.
2064pub fn run_id_for(trigger_hash: &str, connector: Option<&str>, dedup_value: &str) -> String {
2065 let mut h = Sha256::new();
2066 h.update(b"areev-trigger:v1");
2067 h.update([0x1f]);
2068 h.update(connector.unwrap_or_default().as_bytes());
2069 h.update([0x1f]);
2070 h.update(dedup_value.as_bytes());
2071 let digest = hex(&h.finalize());
2072 format!("{}-{}", short(trigger_hash, 8), short(&digest, 12))
2073}
2074
2075fn hex(bytes: &[u8]) -> String {
2076 bytes.iter().map(|b| format!("{b:02x}")).collect()
2077}
2078
2079#[cfg(test)]
2080mod tests {
2081 use super::*;
2082
2083 #[test]
2084 fn a_run_id_is_stable_for_the_same_item() {
2085 let a = run_id_for("abcdef1234", Some("gmail"), "m1");
2086 let b = run_id_for("abcdef1234", Some("gmail"), "m1");
2087 assert_eq!(a, b, "the same item must always mint the same id");
2088 }
2089
2090 #[test]
2091 fn different_items_get_different_ids() {
2092 let a = run_id_for("abcdef1234", Some("gmail"), "m1");
2093 let b = run_id_for("abcdef1234", Some("gmail"), "m2");
2094 assert_ne!(a, b);
2095 }
2096
2097 #[test]
2098 fn the_connector_is_part_of_the_identity() {
2099 // CloudEvents' rule: `id` alone is unique only within a producer, so two
2100 // connectors with a shared id space must not collide.
2101 let a = run_id_for("abcdef1234", Some("gmail"), "1");
2102 let b = run_id_for("abcdef1234", Some("stripe"), "1");
2103 assert_ne!(a, b);
2104 }
2105
2106 #[test]
2107 fn the_id_names_its_trigger_for_a_human_reading_a_run_list() {
2108 let id = run_id_for("abcdef1234567890", Some("gmail"), "m1");
2109 assert!(id.starts_with("abcdef12"), "{id}");
2110 assert!(id.len() < 32, "short enough to read: {id}");
2111 }
2112
2113 #[test]
2114 fn a_short_trigger_hash_does_not_panic() {
2115 let id = run_id_for("ab", None, "x");
2116 assert!(id.starts_with("ab-"));
2117 }
2118}