Skip to main content

areev_trigger/
eval.rs

1//! The evaluation cycle: find due, claim, invoke, ingest, start, release.
2//!
3//! One invocation performs the whole cycle and exits. Nothing stays resident.
4//! Losing a claim race is a no-op exit, not an error — that is the steady state
5//! when several nodes share a heartbeat.
6//!
7//! ## Why correctness does not rest on the lease
8//!
9//! A lease deduplicates *"who is working right now"*. But the thing that
10//! actually needs deduplicating — a firing of trigger T for item I — has a
11//! natural identity, so the run id is derived from it. `areev run start`
12//! already refuses a duplicate run id, so two evaluators racing produce one run
13//! and one recorded skip, with no lease consulted at all: no duration to guess,
14//! no fencing token to compare, no clock-skew window.
15//!
16//! The lease then does one narrower job: stopping two nodes from both making
17//! the same expensive, side-effecting connector call. Losing it can cost a
18//! wasted API call; it can never cause a missed or duplicated firing. That is
19//! the risk profile a lease should carry.
20
21use std::collections::HashMap;
22use std::sync::Arc;
23
24use areev_cal::AreevFacade;
25use areev_core::error::{AreevError, Hash};
26use areev_core::types::{ContentRef, Event, Grain, GrainType, Observation, Trigger, TriggerKind};
27use areev_run::HostToolExecutor;
28use areev_store::TriggerState;
29use serde::Serialize;
30use sha2::{Digest, Sha256};
31
32use crate::clock::Clock;
33use crate::connector::{
34    dedup_value, rewrite_blob_refs, PollItem, PollRequest, PollResponse,
35    MAX_BLOB_BYTES_PER_ITEM, MAX_BLOB_BYTES_PER_RESPONSE,
36};
37use crate::error::{Result, TriggerError};
38use crate::schedule;
39
40/// Namespace for trigger firing records. Alongside the runtime's own
41/// `agent:harness`, so operational journalling stays out of user namespaces.
42pub const TRIGGER_NS: &str = "agent:triggers";
43
44/// How many triggers one invocation will look at. A backstop, not a policy.
45const MAX_TRIGGERS_SCANNED: usize = 4096;
46
47/// Knobs for one evaluation pass.
48#[derive(Debug, Clone)]
49pub struct EvalOptions {
50    /// Evaluate only this trigger (its content address).
51    pub only: Option<String>,
52    /// How long a claim is held before another evaluator may take over.
53    /// Deliberately generous: a lease shorter than the worst-case connector
54    /// runtime causes takeovers mid-poll, which costs duplicate API calls.
55    pub lease: std::time::Duration,
56    /// Cap on items requested per firing.
57    pub max_items: usize,
58    /// Report what would happen and touch nothing. The safe first command on a
59    /// new deployment.
60    pub dry_run: bool,
61    /// This evaluator's identity, recorded in the claim.
62    pub node: String,
63    /// Decoded-size budget for one item's connector blobs (#93). Overrun
64    /// refuses the whole poll (TRG-E011) rather than truncating.
65    pub max_blob_bytes_per_item: usize,
66    /// Decoded-size budget for one poll response's blobs (#93).
67    pub max_blob_bytes_per_response: usize,
68}
69
70impl Default for EvalOptions {
71    fn default() -> Self {
72        EvalOptions {
73            only: None,
74            lease: std::time::Duration::from_secs(300),
75            max_items: 100,
76            dry_run: false,
77            node: default_node_id(),
78            max_blob_bytes_per_item: MAX_BLOB_BYTES_PER_ITEM,
79            max_blob_bytes_per_response: MAX_BLOB_BYTES_PER_RESPONSE,
80        }
81    }
82}
83
84/// First `n` characters, safely — `&s[..n]` panics on a short string or a
85/// multi-byte boundary, and a message-formatting helper must not be what takes
86/// the process down. Declarations can arrive by bundle import from an
87/// implementation we did not write.
88fn short(s: &str, n: usize) -> &str {
89    match s.char_indices().nth(n) {
90        Some((idx, _)) => &s[..idx],
91        None => s,
92    }
93}
94
95/// A stable-ish identity for this evaluator. Host name plus pid: enough to tell
96/// two nodes apart in a claim, and to recognise our own stale claim after a
97/// restart.
98fn default_node_id() -> String {
99    let host = std::env::var("HOSTNAME")
100        .or_else(|_| std::env::var("COMPUTERNAME"))
101        .unwrap_or_else(|_| "node".into());
102    format!("{host}/{}", std::process::id())
103}
104
105/// What one evaluation pass did. Shaped like `loop run`'s report for symmetry.
106#[derive(Debug, Clone, Default, Serialize)]
107pub struct EvalReport {
108    pub claimed: usize,
109    pub skipped_not_due: usize,
110    pub skipped_locked: usize,
111    pub skipped_paused: usize,
112    pub items: usize,
113    pub duplicates: usize,
114    pub runs_started: usize,
115    /// Items recorded without starting a workflow, because no runtime is wired
116    /// in. Counted apart from `runs_started` so the report never claims a run
117    /// happened when none did.
118    pub ingested: usize,
119    /// Items the connector returned that carry no usable identity, so no run
120    /// could be minted for them. Reported rather than silently dropped: an item
121    /// nobody can name is a connector bug, and it should be visible as one.
122    pub unidentifiable: usize,
123    /// Declarations that cannot fire as written — a cron that does not parse, a
124    /// timezone this build refuses, a composite gate naming a member the
125    /// declaration does not carry.
126    ///
127    /// Counted APART from `skipped_not_due` on purpose. Folded in there, an
128    /// unusable trigger is indistinguishable from a healthy one waiting its
129    /// turn, which is the failure mode with no symptom: the work simply never
130    /// happens and every report looks green. The reason lands in `errors`.
131    pub unusable: usize,
132    /// Not fatal to the pass: one broken trigger must not stop the others.
133    pub errors: Vec<String>,
134    /// A connector reported a backlog, so this trigger is due again at once
135    /// rather than after its interval.
136    pub draining: Vec<String>,
137    /// Triggers whose cursor was deliberately left unmoved this pass because
138    /// at least one item in the firing failed to start a run (#129) — the
139    /// Event add failed, declared context was unavailable, or the run
140    /// itself refused. The same page is offered again next tick; whatever
141    /// already started comes back as a `duplicates` count rather than
142    /// double-processing, which is what makes re-polling safe — see #128's
143    /// stable run ids, a prerequisite for this.
144    pub cursor_held: Vec<String>,
145}
146
147/// One trigger considered, for `--dry-run` and `status`.
148#[derive(Debug, Clone, Serialize)]
149pub struct TriggerStatus {
150    pub trigger: String,
151    /// The declaration's `name`, when it carries one.
152    ///
153    /// `name` is what a human uses to identify a trigger, and it is accepted
154    /// on write — but nothing read it back (#73), so identity fell onto the
155    /// workflow hash, which is stable only until the plan is re-declared. It
156    /// rides in `extra_fields` rather than being a typed field: OMS §A.7 has
157    /// no Trigger `name`, and inventing one would be a spec-level decision
158    /// for a label.
159    #[serde(skip_serializing_if = "Option::is_none")]
160    pub name: Option<String>,
161    pub kind: String,
162    pub workflow: String,
163    /// The connector's name, and — when its code is a grain (#185) — the
164    /// Definition that carries it.
165    ///
166    /// Reported because a reader looking at a heartbeat needs to know which of
167    /// the two connector kinds this is: the answer decides whether the code
168    /// travelled with the memory or lives on whichever machine happens to run
169    /// the pass, and that is the first question a stale-poll investigation
170    /// asks.
171    #[serde(skip_serializing_if = "Option::is_none")]
172    pub connector: Option<String>,
173    #[serde(skip_serializing_if = "Option::is_none")]
174    pub connector_tool: Option<String>,
175    pub enabled: bool,
176    pub paused: bool,
177    pub due: bool,
178    /// The IANA zone a SCHEDULE trigger's cron is evaluated in (#297).
179    ///
180    /// `None` for UTC and for every non-schedule kind. Reported because
181    /// "when is this due" now depends on it, and on a DST boundary the UTC
182    /// instant moves by an hour while the local wall time does not — so a
183    /// status surface that showed only the UTC instant would look like the
184    /// schedule had drifted.
185    #[serde(skip_serializing_if = "Option::is_none")]
186    pub timezone: Option<String>,
187    /// The next due instant in the trigger's own zone, beside the UTC one
188    /// (#297). `None` when the zone is UTC or nothing is due.
189    #[serde(skip_serializing_if = "Option::is_none")]
190    pub next_due_local: Option<String>,
191    pub leased_by: Option<String>,
192    pub next_due_at: Option<i64>,
193    pub last_fired_at: Option<i64>,
194    pub consecutive_failures: u32,
195    pub last_error: Option<String>,
196    /// Will never fire again — a one-shot past its instant.
197    pub exhausted: bool,
198    /// Never fired and no failure recorded — the state an unnoticed
199    /// misconfiguration sits in, so it is reported rather than inferred.
200    pub never_fired: bool,
201    /// The plan this trigger names has been EDITED: the current head of its
202    /// supersession chain, when that is not the hash the trigger points at.
203    ///
204    /// A Workflow is content-addressed, so editing one mints a new address and
205    /// a trigger keeps starting the version it was declared against. That is
206    /// the correct default — a plan edit must not silently change what fires
207    /// unattended — but it is invisible without this: nothing in a listing
208    /// distinguished a trigger running the current plan from one running a
209    /// version three edits old, and the run itself succeeds either way.
210    ///
211    /// Reported, never followed. `areev trigger retarget` is the deliberate
212    /// act that moves it, and it is a human's to take.
213    #[serde(skip_serializing_if = "Option::is_none")]
214    pub plan_head: Option<String>,
215    /// Why this declaration can never fire, if it cannot.
216    ///
217    /// A trigger can become unusable without ever being written through a
218    /// validating path — it can arrive by bundle import from an implementation
219    /// that validated differently, or predate the check. So the evaluator
220    /// reports it rather than assuming the write path caught everything.
221    #[serde(skip_serializing_if = "Option::is_none")]
222    pub unusable: Option<String>,
223    /// The connector's last reported position, for a trigger that has ever
224    /// polled. Opaque and connector-defined (#128) — surfaced so an operator
225    /// can SEE the state that a re-pointed trigger's evaluation carries
226    /// forward, which before this was invisible even though nothing else
227    /// could carry it by hand.
228    #[serde(skip_serializing_if = "Option::is_none")]
229    pub cursor: Option<String>,
230    /// The op-log position, a `memory` trigger's cursor equivalent.
231    #[serde(skip_serializing_if = "Option::is_none")]
232    pub op_cursor: Option<i64>,
233    /// The hash evaluation state is actually keyed under: `trg:<chain_root>`
234    /// (#128). Equal to `trigger` for a trigger that has never been
235    /// superseded — the common case, unchanged from before this field
236    /// existed. Differs only for a trigger re-pointed by `SUPERSEDE`, where
237    /// it names the ORIGINAL declaration's hash — the identity the cursor
238    /// and dedup fence actually survive under.
239    pub chain_root: String,
240}
241
242/// The declaration's human label, if it has one.
243///
244/// Written through `extra_fields` by every surface that accepts `name` on a
245/// trigger, and read back here so `list` and `status` can print it (#73).
246/// Blank is treated as absent: a name that renders as nothing is worse than
247/// none, because it displaces the hash a reader could otherwise have used.
248pub fn trigger_name(trigger: &Trigger) -> Option<String> {
249    trigger
250        .common
251        .extra_fields
252        .get("name")
253        .and_then(|v| v.as_str())
254        .map(str::trim)
255        .filter(|s| !s.is_empty())
256        .map(str::to_string)
257}
258
259/// Starts the workflow a trigger is bound to.
260///
261/// A seam rather than a direct `areev_run::Runner` call, for the same reason
262/// the CAL executor takes a governance host: the runtime sits above this crate
263/// in the dependency order for hosts that want it, and a test needs to observe
264/// what *would* be started without spending a real run. It is also where the
265/// duplicate rule lives, which is the whole idempotency story.
266pub trait RunStarter: Send + Sync {
267    fn start(&self, workflow: &str, run_id: &str, input: serde_json::Value) -> StartResult;
268}
269
270/// What starting a run did.
271#[derive(Debug, Clone, PartialEq)]
272pub enum StartResult {
273    Started,
274    /// A run with this id already exists. **Not an error** — it is the
275    /// idempotency guarantee working: this item was already handled, by an
276    /// earlier firing or by another node that won the race.
277    Duplicate,
278    Failed(String),
279}
280
281/// How this host runs a connector that is a **grain** rather than a command
282/// (#185).
283///
284/// Host config, never in the file, for exactly the reason `--allow-executor`
285/// is host config on the run path: a bundle carries the connector's code, so a
286/// permission arriving in the same bundle as the code it authorizes is not a
287/// permission. A trigger may name any Definition it likes; nothing runs unless
288/// this host pinned its address.
289///
290/// Absent (`None` on the evaluator) means a trigger naming a connector
291/// Definition refuses with `TRG-E012` — the same fail-closed shape as a due
292/// polling trigger with no `--connector-cmd`.
293#[derive(Debug, Clone, Default)]
294pub struct ConnectorCode {
295    /// Content addresses this host will execute — `--allow-executor`, the
296    /// same comma list and the same meaning the run path gives it.
297    pub allow: Vec<String>,
298    /// Where a pinned blob is materialized (`--executor-cache`).
299    pub cache_dir: Option<std::path::PathBuf>,
300    /// The `areev-sandbox` runner, argv-split by the executor. Required for a
301    /// `wasm32-areev`/`wasm32-areev-io` connector, which is the shape this
302    /// exists for; a native pinned blob runs without one.
303    pub sandbox_cmd: Option<String>,
304    /// Per-invocation wall-clock ceiling (`--executor-timeout`), in the run
305    /// path's tri-state: `None` leaves the executor's own default standing,
306    /// `Some(None)` waits forever on request, `Some(Some(d))` overrides.
307    pub timeout: Option<Option<std::time::Duration>>,
308    /// Environment policy for a NATIVE pinned blob. The sandbox clears its
309    /// environment regardless — see `CodeExecutor`.
310    pub env: Option<areev_core::proc::EnvPolicy>,
311    /// The memory's locator, so a connector declaring `{"blob": {"read":
312    /// true}}` can be served by the per-poll broker. The evaluator holds the
313    /// memory while a connector runs, and the broker's blob read deliberately
314    /// does not open it — it goes to the `.blobs` sidecar, or to one
315    /// short-lived Postgres connection of its own (#202).
316    pub db_locator: Option<String>,
317}
318
319/// A connector Definition resolved, pinned and read — everything
320/// `execute_code` needs, gathered before a broker is started so a refusal
321/// costs no side effects.
322struct PreparedConnector {
323    /// The Definition's own content address, which is what the trigger names.
324    tool_hash: String,
325    code: areev_run::PreparedCode,
326    /// True when the pinned declaration asks to read CAS blobs, which is what
327    /// decides whether the per-poll broker serves them.
328    wants_blobs: bool,
329    /// The Definition's `config` (#231) — the connector's WIRING, as opposed
330    /// to the trigger's `config`, which is the instance. A generic blob like
331    /// `rest.poll` is made provider-specific here, beside the `capabilities`
332    /// block it has to agree with, and travels with the code rather than
333    /// with each declaration that names it.
334    config: Option<serde_json::Value>,
335}
336
337/// The `config` a connector is handed: the Definition's wiring with the
338/// trigger's own keys laid over it.
339///
340/// Shallow, and the trigger wins — the more specific instance specializes the
341/// shared declaration, which is what lets two mailboxes share one Definition.
342/// Either side alone passes through untouched, so a connector that predates
343/// Definition config sees exactly what it saw before.
344fn merged_config(
345    definition: Option<&serde_json::Value>,
346    trigger: Option<&serde_json::Value>,
347) -> Option<serde_json::Value> {
348    match (definition, trigger) {
349        (None, t) => t.cloned(),
350        (Some(d), None) => Some(d.clone()),
351        (Some(d), Some(t)) => match (d.as_object(), t.as_object()) {
352            (Some(d), Some(t)) => {
353                let mut out = d.clone();
354                for (k, v) in t {
355                    out.insert(k.clone(), v.clone());
356                }
357                Some(serde_json::Value::Object(out))
358            }
359            // Neither is an object: nothing to merge key-wise, and the
360            // instance is the more specific statement.
361            _ => Some(t.clone()),
362        },
363    }
364}
365
366/// `cas://sha256:<hex>` and a bare `<hex>` are the same pin, compared the way
367/// the executor compares them.
368fn normalize_address(uri: &str) -> String {
369    uri.trim()
370        .strip_prefix("cas://sha256:")
371        .unwrap_or_else(|| uri.trim())
372        .to_ascii_lowercase()
373}
374
375/// The inner executor a connector's [`areev_run::CodeExecutor`] wraps.
376///
377/// It is never reached: this path only ever calls `execute_code`, which
378/// dispatches the blob itself. It exists because `CodeExecutor` wraps
379/// something by construction, and what it wraps on the run path — the host's
380/// `--tool-cmd` — must NOT be silently reachable here. A connector that
381/// resolved to a command would be a different program than the declaration
382/// names.
383struct NoConnectorFallback;
384
385impl HostToolExecutor for NoConnectorFallback {
386    fn execute(
387        &self,
388        tool_name: &str,
389        _hash: &str,
390        _input: &serde_json::Value,
391        _idem: &str,
392    ) -> areev_run::ExecResult {
393        areev_run::ExecResult::Err {
394            cause: areev_run::FailCause::ExecutorError,
395            detail: format!(
396                "connector {tool_name:?} is declared as a grain, so it has no host command to \
397                 fall back to"
398            ),
399        }
400    }
401}
402
403/// Evaluates triggers against one memory.
404pub struct Evaluator {
405    pub facade: Arc<AreevFacade>,
406    pub clock: Arc<dyn Clock>,
407    /// Executes connector commands. `None` means a due polling trigger fails
408    /// loudly with `TRG-E003` rather than quietly doing nothing — a poll that
409    /// silently returns no items is indistinguishable from a healthy source.
410    pub connector: Option<Arc<dyn HostToolExecutor>>,
411    /// How a connector that is a GRAIN runs (#185). `None` means this host
412    /// runs host-command connectors only, and a trigger naming a connector
413    /// Definition refuses with `TRG-E012` rather than falling back to
414    /// `--connector-cmd` — a fallback would run a different program than the
415    /// declaration names, which is the failure with no symptom the pin exists
416    /// to refuse.
417    pub connector_code: Option<ConnectorCode>,
418    /// Starts the bound workflow. `None` ingests the item and records the
419    /// firing without starting anything — what `--no-start` is for, and what
420    /// the ingest-only tests use.
421    pub starter: Option<Arc<dyn RunStarter>>,
422    /// Credentials the broker may attach, by name. Values live here and never
423    /// in a grain or a connector's environment: a declaration names a
424    /// credential, it never carries one.
425    /// A SOURCE per name, not a resolved value (#113): the heartbeat is the
426    /// path that most needs short-lived credentials, because it is the one
427    /// nobody is watching. An env-var source resolves once, as before; a
428    /// `cmd:`/`vault:` source is minted per TTL window inside the broker, so a
429    /// cadence that outlives a token's hour does not start 401'ing overnight.
430    pub credentials: std::collections::BTreeMap<String, areev_run::CredentialSource>,
431    pub ns: String,
432    pub principal: String,
433}
434
435impl Evaluator {
436    /// An evaluator that can inspect but not act: no connector, no runtime, no
437    /// credentials. What `list`, `show` and `status` want, and the shape that
438    /// makes "reading cannot fire anything" true by construction rather than by
439    /// remembering to pass `None` four times.
440    pub fn read_only(facade: Arc<AreevFacade>, clock: Arc<dyn Clock>, ns: &str) -> Evaluator {
441        Evaluator {
442            facade,
443            clock,
444            connector: None,
445            connector_code: None,
446            starter: None,
447            credentials: Default::default(),
448            ns: ns.to_string(),
449            principal: "user:local".into(),
450        }
451    }
452
453    /// Every trigger declaration in the memory, newest first.
454    pub fn declarations(&self) -> Result<Vec<(String, Trigger)>> {
455        let ns = self.ns.clone();
456        let grains = self
457            .facade
458            .with_store(|m| {
459                m.recent_live_scoped(&[ns], Some(GrainType::Trigger), MAX_TRIGGERS_SCANNED)
460            })
461            .map_err(|e| TriggerError::Storage { detail: e.to_string() })?;
462
463        let mut out = Vec::new();
464        for g in grains {
465            let hash = g.hash.to_hex();
466            match g.to_trigger() {
467                Ok(mut t) => {
468                    // `to_trigger` reconstructs the typed fields; `extra_fields`
469                    // is not restored for any grain type, so the declaration's
470                    // `name` — an extra field by design, since OMS §A.7 has no
471                    // Trigger `name` — has to be carried over here. Doing it in
472                    // the shared deserializer would silently change what every
473                    // read-modify-write re-serializes, which is the kind of
474                    // change that moves content addresses.
475                    if let Some(name) = g.get_str("name") {
476                        t.common
477                            .extra_fields
478                            .insert("name".into(), serde_json::json!(name));
479                    }
480                    out.push((hash, t))
481                }
482                // A declaration this build cannot read is reported, never
483                // skipped silently: silence is the symptom of every trigger
484                // failure, so it must never also be the symptom of a bug.
485                Err(e) => {
486                    return Err(TriggerError::Malformed {
487                        what: format!("declaration {hash} is unreadable: {e}"),
488                    })
489                }
490            }
491        }
492        Ok(out)
493    }
494
495    /// What every trigger's state looks like right now.
496    pub fn status(&self) -> Result<Vec<TriggerStatus>> {
497        let now = self.clock.now_ms();
498        let mut out = Vec::new();
499        // One cache for this call, same as a `run()` pass — a chain is
500        // walked at most once even though `declarations()` above may list
501        // the same workflow's triggers repeatedly.
502        let mut chains: HashMap<String, Chain> = HashMap::new();
503        for (hash, t) in self.declarations()? {
504            let chain = self.resolve_chain(&hash, &mut chains)?;
505            // Read-only: `adopt = false`. `trigger status`/`show` must not
506            // have the side effect of migrating a legacy state row — a real
507            // evaluation pass (`run`/`deliver`) does that; this only
508            // reports whatever it finds, root row or legacy.
509            let (st, _raw) = self.resolve_state(&chain, false)?;
510            let unusable = schedule::validate(&t).err().map(|e| e.to_string());
511            let plan_head = self.plan_head_if_stale(&t)?;
512            out.push(TriggerStatus {
513                trigger: hash,
514                name: trigger_name(&t),
515                kind: t.kind.as_str().to_string(),
516                workflow: t.workflow.clone(),
517                connector: t.connector.clone(),
518                connector_tool: t.connector_tool.clone(),
519                enabled: t.enabled,
520                paused: st.paused,
521                // An unusable declaration is never due. Saying otherwise would
522                // promise a firing that cannot happen.
523                due: unusable.is_none()
524                    && t.enabled
525                    && !st.leased(now)
526                    && schedule::is_due(&t, &st, now).unwrap_or(false),
527                timezone: schedule::declared_zone(&t),
528                next_due_local: st
529                    .next_due_at
530                    .and_then(|at| schedule::render_local(&t, at)),
531                leased_by: st.leased(now).then(|| st.claimed_by.clone()).flatten(),
532                next_due_at: st.next_due_at,
533                last_fired_at: st.last_fired_at,
534                consecutive_failures: st.consecutive_failures,
535                last_error: st.last_error.clone(),
536                exhausted: st.exhausted,
537                never_fired: st.last_fired_at.is_none() && st.consecutive_failures == 0,
538                unusable,
539                cursor: st.cursor.clone(),
540                op_cursor: st.op_cursor,
541                plan_head,
542                chain_root: chain.root().to_string(),
543            });
544        }
545        Ok(out)
546    }
547
548    /// Ingest a payload delivered by the host for a `webhook` or `manual`
549    /// trigger.
550    ///
551    /// Areev never opens a port. The host already terminates TLS and
552    /// authenticates the sender — it is far better at both than a memory engine
553    /// would be — and hands the payload to a one-shot invocation. That keeps the
554    /// useful half of webhooks without a listening daemon.
555    ///
556    /// Idempotent on the same terms as a poll: the payload's dedup value mints
557    /// the run id, so a webhook delivered twice (which every provider does)
558    /// produces one run and one recorded skip.
559    pub fn deliver(&self, trigger_hash: &str, payload: serde_json::Value) -> Result<EvalReport> {
560        let now = self.clock.now_ms();
561        let (hash, trigger) = self
562            .declarations()?
563            .into_iter()
564            .find(|(h, _)| h.starts_with(trigger_hash))
565            .ok_or_else(|| TriggerError::Malformed {
566                what: format!("no trigger matching '{trigger_hash}'"),
567            })?;
568
569        if !matches!(trigger.kind, TriggerKind::Webhook | TriggerKind::Manual) {
570            return Err(TriggerError::Malformed {
571                what: format!(
572                    "trigger {} is a {} trigger — only webhook and manual triggers accept a \
573                     delivery; a clocked or polling trigger fires from `trigger run`",
574                    short(&hash, 12),
575                    trigger.kind.as_str()
576                ),
577            });
578        }
579        if !trigger.enabled {
580            return Err(TriggerError::Malformed {
581                what: format!("trigger {} is disabled", short(&hash, 12)),
582            });
583        }
584        // #128 — a single-trigger cache: `deliver` fires exactly one
585        // trigger, so this walks its chain at most once regardless of how
586        // many places below need the root.
587        let mut chains: HashMap<String, Chain> = HashMap::new();
588        let chain = self.resolve_chain(&hash, &mut chains)?;
589        let (state, _raw) = self.resolve_state(&chain, true)?;
590        if state.paused {
591            return Err(TriggerError::Malformed {
592                what: format!("trigger {} is paused", short(&hash, 12)),
593            });
594        }
595
596        let mut report = EvalReport::default();
597        let item = PollItem { id: String::new(), payload, blobs: Vec::new() };
598        report.items = 1;
599        let Some(value) = dedup_value(&item, &trigger.dedup_key) else {
600            report.unidentifiable = 1;
601            // Journal before returning. A delivery naming nothing is precisely
602            // the case the record exists for: the sender's payload shape
603            // changed under us, and stdout of whatever invoked this is not an
604            // audit record -- the replicating Observation is.
605            self.journal(
606                &hash,
607                &trigger,
608                &FireOutcome { items: 1, unidentifiable: 1, ..Default::default() },
609                Some("delivered"),
610                now,
611            )?;
612            return Ok(report);
613        };
614        // Keyed on the chain ROOT (#128), so a delivery replayed after the
615        // trigger has been re-pointed by `SUPERSEDE` still mints the same id
616        // it always would have. `legacy_run_ids` covers a trigger already
617        // superseded before this fix — the id it would have minted under
618        // its old (then-current) head.
619        let root = chain.root();
620        let run_id = run_id_for(root, trigger.connector.as_deref(), &value);
621        let legacy_run_ids: Vec<String> = chain
622            .legacy()
623            .iter()
624            .map(|h| run_id_for(h, trigger.connector.as_deref(), &value))
625            .collect();
626        match self.start_run(&trigger, &run_id, &legacy_run_ids, &item, &hash, &[]) {
627            StartOutcome::Started => report.runs_started = 1,
628            StartOutcome::Ingested => report.ingested = 1,
629            StartOutcome::Duplicate => report.duplicates = 1,
630            StartOutcome::Failed(why) => report.errors.push(why),
631        }
632
633        let outcome = FireOutcome {
634            items: 1,
635            runs_started: report.runs_started,
636            duplicates: report.duplicates,
637            ..Default::default()
638        };
639        self.journal(&hash, &trigger, &outcome, Some("delivered"), now)?;
640        Ok(report)
641    }
642
643    /// Run one evaluation pass.
644    pub fn run(&self, opts: &EvalOptions) -> Result<EvalReport> {
645        let mut report = EvalReport::default();
646        let now = self.clock.now_ms();
647        // #128 — resolved once per trigger per PASS, never once per item:
648        // every chain walk below goes through this cache, shared with
649        // `record_for_composites` (a composite may be touched by several
650        // members firing in the same pass).
651        let mut chains: HashMap<String, Chain> = HashMap::new();
652
653        let declarations = self.declarations()?;
654        // Composites are settled after their members, so a gate can be
655        // satisfied in the same pass that completes it rather than waiting a
656        // whole heartbeat. Declaration order is newest-first and says nothing
657        // about dependency, so the split is explicit.
658        let (composites, members): (Vec<_>, Vec<_>) =
659            declarations.into_iter().partition(|(_, t)| t.kind == TriggerKind::Composite);
660
661        for (hash, trigger) in members.into_iter().chain(composites) {
662            if opts.only.as_ref().is_some_and(|only| only != &hash) {
663                continue;
664            }
665            if !trigger.enabled {
666                report.skipped_not_due += 1;
667                continue;
668            }
669            // Checked BEFORE dueness, so an unusable declaration is reported as
670            // what it is rather than disappearing into `not due` — where it
671            // looks exactly like a healthy trigger waiting its turn.
672            if let Err(why) = schedule::validate(&trigger) {
673                report.unusable += 1;
674                report.errors.push(format!("{}: {why}", short(&hash, 12)));
675                continue;
676            }
677            // #128 — every read/write of this trigger's evaluation state
678            // below is keyed on the chain ROOT, not on `hash` (the
679            // declaration's current head): a trigger re-pointed by
680            // `SUPERSEDE` mints a new head for the same standing rule, and
681            // keying state on the head alone would orphan the cursor and
682            // the dedup fence on every applied recommendation. For a
683            // never-superseded trigger root == hash, so this resolves to
684            // exactly the lookup this crate always did — a byte-identical
685            // no-op (pinned by `a_never_superseded_trigger_keys_state_on_its_own_hash`).
686            // `adopt = true`: this is a real evaluation pass with write
687            // access, so a legacy (pre-#128, head-keyed) row found along the
688            // way is migrated under the root.
689            let chain = self.resolve_chain(&hash, &mut chains)?;
690            let (state, raw) = self.resolve_state(&chain, true)?;
691
692            if state.paused {
693                report.skipped_paused += 1;
694                continue;
695            }
696            // An absolute schedule seen for the first time: record when it
697            // will be due rather than re-deriving it every pass, so `trigger
698            // status` can say when it next fires and the comparison afterwards
699            // is a plain timestamp check.
700            if state.next_due_at.is_none() && !state.exhausted {
701                if let Some(first) = schedule::initial_due(&trigger, now)? {
702                    let mut seeded = state.clone();
703                    seeded.next_due_at = Some(first);
704                    let _ = self
705                        .facade
706                        .with_store(|m| m.put_trigger_state(chain.root(), raw.as_deref(), &seeded))
707                        .map_err(|e| TriggerError::Storage { detail: e.to_string() })?;
708                    report.skipped_not_due += 1;
709                    continue;
710                }
711            }
712            if !schedule::is_due(&trigger, &state, now)? {
713                report.skipped_not_due += 1;
714                continue;
715            }
716            if state.leased(now) {
717                // The normal steady state with several nodes on one heartbeat,
718                // not a problem.
719                report.skipped_locked += 1;
720                continue;
721            }
722            if opts.dry_run {
723                report.claimed += 1;
724                continue;
725            }
726
727            match self.fire(&hash, &chain, &trigger, state, raw.as_deref(), now, opts) {
728                // Lost the claim between deciding it was free and taking it.
729                // Normal with several nodes on one heartbeat, and it must not
730                // be counted as work done.
731                Ok(outcome) if outcome.skipped_locked => report.skipped_locked += 1,
732                Ok(outcome) => {
733                    if !outcome.fired_items.is_empty() {
734                        if let Err(e) =
735                            self.record_for_composites(&hash, &outcome.fired_items, now, &mut chains)
736                        {
737                            report.errors.push(e.to_string());
738                        }
739                    }
740                    report.claimed += 1;
741                    report.items += outcome.items;
742                    report.duplicates += outcome.duplicates;
743                    report.runs_started += outcome.runs_started;
744                    report.ingested += outcome.ingested;
745                    report.unidentifiable += outcome.unidentifiable;
746                    for why in &outcome.failures {
747                        report.errors.push(format!("{hash}: {why}"));
748                    }
749                    // #129 — any start failure in this firing means the
750                    // cursor was deliberately left unmoved (see `fire`'s
751                    // release logic); surfaced apart from `errors` so an
752                    // operator sees WHY the same items come back next tick
753                    // rather than inferring it.
754                    if !outcome.failures.is_empty() {
755                        report.cursor_held.push(hash.clone());
756                    }
757                    if outcome.draining {
758                        report.draining.push(hash.clone());
759                    }
760                }
761                Err(e) => {
762                    // One broken trigger must not stop the others.
763                    report.errors.push(e.to_string());
764                }
765            }
766        }
767        Ok(report)
768    }
769
770    // `chain` (#128) is one more independent knob on an already knob-heavy
771    // internal call, in the same spirit as the other `too_many_arguments`
772    // allows in this codebase (areev-store's tuned-recall internals, the
773    // bindings' flat FFI surfaces) — splitting it into a struct would not
774    // make any single call site more readable.
775    #[allow(clippy::too_many_arguments)]
776    fn fire(
777        &self,
778        hash: &str,
779        chain: &Chain,
780        trigger: &Trigger,
781        state: TriggerState,
782        raw: Option<&str>,
783        now: i64,
784        opts: &EvalOptions,
785    ) -> Result<FireOutcome> {
786        // #128 — the claim/release CAS below is keyed on the chain ROOT, not
787        // `hash`. This is the hazard the fix has to get right: `fire` claims
788        // with one key and releases with the same key, so if the two ever
789        // disagreed about which key that is, the release's compare-and-swap
790        // would find no row, `released_ok` would be false, and every firing
791        // of a superseded trigger would permanently report `ClaimLost` — a
792        // trigger that could never usefully claim again. `root` is resolved
793        // exactly once here and reused for every claim/release call in this
794        // function.
795        let root = chain.root();
796
797        // Claim. Losing means another evaluator got here first.
798        let mut claimed = state.clone();
799        claimed.fence = claimed.fence.wrapping_add(1);
800        claimed.claimed_by = Some(opts.node.clone());
801        claimed.lease_until = Some(now + opts.lease.as_millis() as i64);
802        let won = self
803            .facade
804            .with_store(|m| m.put_trigger_state(root, raw, &claimed))
805            .map_err(|e| TriggerError::Storage { detail: e.to_string() })?;
806        if !won {
807            return Ok(FireOutcome { skipped_locked: true, ..Default::default() });
808        }
809        // What a later release must present as `expected`.
810        let claimed_raw = self
811            .facade
812            .with_store(|m| m.trigger_state(root))
813            .map_err(|e| TriggerError::Storage { detail: e.to_string() })?
814            .map(|(_, r)| r);
815
816        let result = self.collect_and_start(hash, chain, trigger, &claimed, now, opts);
817
818        // Release under the fence, whatever happened.
819        let mut released = claimed.clone();
820        released.claimed_by = None;
821        released.lease_until = None;
822        match &result {
823            // Full success: every item that resolved an identity started (or
824            // was ingested, or was recognized as a duplicate) cleanly.
825            Ok(outcome) if outcome.failures.is_empty() => {
826                released.last_fired_at = Some(now);
827                released.consecutive_failures = 0;
828                released.last_error = None;
829                if let Some(c) = &outcome.cursor {
830                    released.cursor = Some(c.clone());
831                }
832                if let Some(c) = outcome.op_cursor {
833                    released.op_cursor = Some(c);
834                }
835                // A satisfied gate consumes its correlation key. Leaving it
836                // would re-fire the same set on every pass; Flink calls this
837                // the after-match skip strategy, and the conservative choice —
838                // clear only the key that fired — is the one that cannot
839                // swallow an unrelated in-flight match.
840                for k in &outcome.settled_keys {
841                    released.partials.remove(k);
842                }
843                prune_partials(&mut released, trigger, now);
844                released.exhausted = schedule::exhausts_after(trigger, now);
845                released.next_due_at = if outcome.draining {
846                    // A backlog drains at once rather than waiting out the
847                    // interval — a cold start should not take a day.
848                    Some(now)
849                } else {
850                    schedule::advance_after_firing(trigger, now, now)?
851                };
852            }
853            // #129 — at least one item failed to start a run: the Event add
854            // failed, declared context was unavailable, or the run itself
855            // refused (RUN-E018 and friends). All three mean the item was
856            // NOT processed, so this firing must not report itself clean —
857            // that is the opposite posture from the Err arm below, which
858            // this deliberately mirrors:
859            //   - the cursor / op_cursor are left at `claimed`'s values
860            //     (never assigned from `outcome`), so the next pass re-asks
861            //     the connector for the same page instead of skipping past
862            //     unprocessed work;
863            //   - `exhausted` is left at `claimed.exhausted` (false, since a
864            //     due trigger cannot already be exhausted) rather than
865            //     recomputed — a `once` trigger that fails must be allowed
866            //     to retry, not be marked as having had its one shot;
867            //   - `settled_keys` are NOT removed from `partials` — a
868            //     satisfied composite gate whose run refused must stay
869            //     satisfied so it retries, not lose its correlation.
870            // Re-polling the same page is safe BECAUSE run ids are keyed on
871            // the chain root (#128): whatever already started is protected
872            // by `start_run`'s dedup fence and comes back as `Duplicate`,
873            // never a second run. What DID happen is still real —
874            // `runs_started`/`ingested`/`duplicates` in the outcome are not
875            // discarded, only the durable cursor advance is withheld.
876            Ok(outcome) => {
877                released.last_fired_at = Some(now);
878                released.consecutive_failures = claimed.consecutive_failures.saturating_add(1);
879                released.last_error = Some(summarize_failures(&outcome.failures));
880                released.next_due_at =
881                    Some(schedule::backoff_until(trigger, released.consecutive_failures, now));
882            }
883            Err(e) => {
884                released.consecutive_failures = claimed.consecutive_failures.saturating_add(1);
885                released.last_error = Some(e.to_string());
886                released.next_due_at =
887                    Some(schedule::backoff_until(trigger, released.consecutive_failures, now));
888            }
889        }
890        let released_ok = self
891            .facade
892            .with_store(|m| m.put_trigger_state(root, claimed_raw.as_deref(), &released))
893            .map_err(|e| TriggerError::Storage { detail: e.to_string() })?;
894        if !released_ok {
895            // Our lease expired mid-firing and someone else took over. Their
896            // state stands; ours is refused. The work we did is not lost — the
897            // runs we started are journaled and idempotent — but we must not
898            // write back over the new owner.
899            self.journal(hash, trigger, &FireOutcome::default(), Some("claim lost"), now)?;
900            return Err(TriggerError::ClaimLost { trigger: hash.to_string() });
901        }
902
903        let outcome = result?;
904        self.journal(hash, trigger, &outcome, None, now)?;
905        Ok(outcome)
906    }
907
908    /// Get the items for this firing and start a run per item.
909    fn collect_and_start(
910        &self,
911        hash: &str,
912        chain: &Chain,
913        trigger: &Trigger,
914        state: &TriggerState,
915        now: i64,
916        opts: &EvalOptions,
917    ) -> Result<FireOutcome> {
918        let mut outcome = FireOutcome::default();
919
920        let items: Vec<PollItem> = match trigger.kind {
921            TriggerKind::Polling => {
922                let response = self.poll(hash, trigger, state, opts)?;
923                outcome.cursor = response.cursor.clone();
924                outcome.draining = response.more;
925                // First contact seeds the cursor and fires nothing. Otherwise
926                // declaring a mailbox trigger starts a run for every message in
927                // history — Zapier's priming poll, and for the same reason.
928                if state.cursor.is_none() {
929                    outcome.seeded = true;
930                    return Ok(outcome);
931                }
932                response.items
933            }
934            // A clocked trigger with no external source fires once per
935            // occurrence, carrying the occurrence itself as the item.
936            TriggerKind::Interval | TriggerKind::Schedule | TriggerKind::Once => {
937                vec![PollItem {
938                    id: now.to_string(),
939                    payload: serde_json::json!({ "at_ms": now }),
940                    blobs: Vec::new(),
941                }]
942            }
943            TriggerKind::Memory => {
944                // Recorded so the journal says "seeded" rather than leaving a
945                // firing that produced nothing looking like a quiet memory.
946                outcome.seeded = state.op_cursor.is_none();
947                let (found, cursor) = self.scan_memory(hash, trigger, state, opts)?;
948                outcome.op_cursor = Some(cursor);
949                found
950            }
951            TriggerKind::Composite => {
952                let keys = self.satisfied_keys(hash, chain, trigger, now)?;
953                outcome.settled_keys = keys.clone();
954                keys.into_iter()
955                    .map(|k| PollItem {
956                        // The correlation value IS the identity: one firing per
957                        // correlated set, however many members contributed.
958                        id: k.clone(),
959                        payload: serde_json::json!({ "correlation": k }),
960                        blobs: Vec::new(),
961                    })
962                    .collect()
963            }
964            // These do not poll. The host owns the listener and hands the
965            // payload to `deliver`, which fires them on the same idempotency
966            // terms; an evaluation pass simply has nothing to do for one.
967            TriggerKind::Webhook | TriggerKind::Manual => Vec::new(),
968        };
969
970        // #93 — store connector blobs in the CAS and rewrite `"blob": "@N"`
971        // payload references to their addresses BEFORE identity resolution,
972        // so a `--dedup-key` pointer into a rewritten field sees the stable
973        // content address rather than a positional index. A contract
974        // violation fails the whole poll here (TRG-E011): fire() then leaves
975        // the cursor unmoved, so nothing is lost — the connector gets fixed
976        // and the same page is re-polled.
977        let mut items = items;
978        let item_refs = self.ingest_item_blobs(hash, &mut items, opts)?;
979
980        outcome.items = items.len();
981        for (item, refs) in items.into_iter().zip(item_refs) {
982            let Some(value) = dedup_value(&item, &trigger.dedup_key) else {
983                outcome.unidentifiable += 1;
984                continue;
985            };
986            outcome.fired_items.push(item.payload.clone());
987            // Keyed on the chain ROOT (#128): the same item seen again after
988            // this trigger has been re-pointed by `SUPERSEDE` still mints
989            // the id it always would have, so `start_run`'s dedup fence
990            // still recognizes it. `legacy_run_ids` is the compatibility
991            // path for a trigger already superseded BEFORE this fix — the
992            // id(s) it would have minted under an older head — and is empty
993            // (so free) for the common never-superseded case.
994            let root = chain.root();
995            let run_id = run_id_for(root, trigger.connector.as_deref(), &value);
996            let legacy_run_ids: Vec<String> = chain
997                .legacy()
998                .iter()
999                .map(|h| run_id_for(h, trigger.connector.as_deref(), &value))
1000                .collect();
1001            match self.start_run(trigger, &run_id, &legacy_run_ids, &item, hash, &refs) {
1002                StartOutcome::Started => outcome.runs_started += 1,
1003                StartOutcome::Ingested => outcome.ingested += 1,
1004                StartOutcome::Duplicate => outcome.duplicates += 1,
1005                StartOutcome::Failed(why) => outcome.failures.push(why),
1006            }
1007        }
1008        Ok(outcome)
1009    }
1010
1011    /// Note a member firing against every composite that declares it.
1012    ///
1013    /// The correlation value comes from the composite's own `correlate`
1014    /// pointer applied to the member's item — so two composites may correlate
1015    /// the same member on different fields, which is why this is resolved per
1016    /// composite rather than once per member.
1017    fn record_for_composites(
1018        &self,
1019        member: &str,
1020        items: &[serde_json::Value],
1021        now: i64,
1022        chains: &mut HashMap<String, Chain>,
1023    ) -> Result<()> {
1024        for (chash, composite) in self.declarations()? {
1025            let Some(alias) = composite.member_alias(member).map(str::to_string) else {
1026                continue;
1027            };
1028            if composite.kind != TriggerKind::Composite {
1029                continue;
1030            }
1031            // #128 — the composite's OWN chain, not the member's: state
1032            // lives under the composite's root regardless of which member
1033            // just fired. Shares the pass-wide cache, so a composite with
1034            // several members touched this pass walks its chain once.
1035            let chain = self.resolve_chain(&chash, chains)?;
1036            let (mut state, raw) = self.resolve_state(&chain, true)?;
1037
1038            for item in items {
1039                // No correlate pointer means one global match — every member
1040                // firing counts toward the same gate.
1041                let key = match &composite.correlate {
1042                    Some(ptr) => match item.pointer(ptr) {
1043                        Some(serde_json::Value::String(s)) => s.clone(),
1044                        Some(other) => other.to_string(),
1045                        // An item that cannot be correlated cannot join a
1046                        // correlated gate. Dropping it is right: attributing it
1047                        // to an arbitrary key would pair unrelated work.
1048                        None => continue,
1049                    },
1050                    None => String::new(),
1051                };
1052                let entry = state.partials.entry(key).or_insert_with(|| {
1053                    areev_store::PartialMatch { members: Default::default(), started_ms: now }
1054                });
1055                // Recorded under the ALIAS, because that is what the gate
1056                // expression names.
1057                entry.members.insert(alias.clone(), now);
1058            }
1059            prune_partials(&mut state, &composite, now);
1060            self.facade
1061                .with_store(|m| m.put_trigger_state(chain.root(), raw.as_deref(), &state))
1062                .map_err(|e| TriggerError::Storage { detail: e.to_string() })?;
1063        }
1064        Ok(())
1065    }
1066
1067    /// Which correlation keys currently satisfy this composite's gate.
1068    fn satisfied_keys(
1069        &self,
1070        hash: &str,
1071        chain: &Chain,
1072        trigger: &Trigger,
1073        now: i64,
1074    ) -> Result<Vec<String>> {
1075        let predicate = trigger
1076            .predicate
1077            .as_ref()
1078            .ok_or_else(|| TriggerError::Malformed {
1079                what: format!("composite trigger {hash} has no predicate"),
1080            })
1081            .and_then(crate::predicate::from_value)?;
1082
1083        // Refuse a gate that names a member the declaration does not carry: it
1084        // could never be satisfied, so the trigger would be silently dead.
1085        for referenced in crate::predicate::referenced_members(&predicate) {
1086            if !trigger.members.contains_key(&referenced) {
1087                return Err(TriggerError::UnknownMember {
1088                    trigger: hash.to_string(),
1089                    member: referenced,
1090                });
1091            }
1092        }
1093
1094        // #128 — this composite's own partials live under its chain root.
1095        // No adoption dance needed here: by the time `fire` reaches
1096        // `collect_and_start` for this trigger, `run()`'s loop has already
1097        // resolved (and, if needed, adopted) this exact root's state row.
1098        let (state, _) = self
1099            .facade
1100            .with_store(|m| m.trigger_state(chain.root()))
1101            .map_err(|e| TriggerError::Storage { detail: e.to_string() })?
1102            .unwrap_or_default();
1103
1104        let mut out = Vec::new();
1105        for (key, partial) in &state.partials {
1106            if let Some(w) = trigger.window_ms {
1107                if now - partial.started_ms > w {
1108                    continue;
1109                }
1110            }
1111            let fired: Vec<String> = partial.members.keys().cloned().collect();
1112            if crate::predicate::gate_satisfied(&predicate, &fired) {
1113                out.push(key.clone());
1114            }
1115        }
1116        Ok(out)
1117    }
1118
1119    /// Grains matching the predicate that appeared since the stored op cursor.
1120    ///
1121    /// Built on the op-log rather than a scan: `changes_since` is an indexed
1122    /// primary-key range read with a monotonic exclusive cursor, so cost is
1123    /// proportional to what changed rather than to how much the memory holds.
1124    ///
1125    /// Considered and rejected: `PRAGMA data_version` as a cheap "did anything
1126    /// change" pre-check. It is documented as unchanged for commits made on the
1127    /// same connection — and Areev is single-writer with one connection, so our
1128    /// own writes are exactly what it cannot see.
1129    fn scan_memory(
1130        &self,
1131        hash: &str,
1132        trigger: &Trigger,
1133        state: &TriggerState,
1134        opts: &EvalOptions,
1135    ) -> Result<(Vec<PollItem>, i64)> {
1136        let predicate = trigger
1137            .predicate
1138            .as_ref()
1139            .ok_or_else(|| TriggerError::Malformed {
1140                what: format!("memory trigger {hash} has no predicate"),
1141            })
1142            .and_then(crate::predicate::from_value)?;
1143
1144        // First contact seeds at the head of the log and matches nothing.
1145        // Otherwise declaring a memory trigger on an established memory fires
1146        // once for every historical grain that matches — the same reason a
1147        // polling trigger primes rather than replaying a mailbox.
1148        let Some(after) = state.op_cursor else {
1149            let head = self
1150                .facade
1151                .with_store(|m| m.head_op_seq())
1152                .map_err(|e| TriggerError::Storage { detail: e.to_string() })?;
1153            return Ok((Vec::new(), head));
1154        };
1155        let ops = self
1156            .facade
1157            .with_store(|m| m.changes_since(after, opts.max_items))
1158            .map_err(|e| TriggerError::Storage { detail: e.to_string() })?;
1159
1160        let mut items = Vec::new();
1161        let mut cursor = after;
1162        for op in ops {
1163            cursor = cursor.max(op.op_seq);
1164            // Additions and supersessions both mean "this is true now".
1165            // A tombstone means the grain is gone, so there is nothing to match.
1166            if op.op == areev_store::OP_FORGET {
1167                continue;
1168            }
1169            let Ok(grain) = self.facade.with_store(|m| m.get(&op.hash)) else {
1170                // A grain that vanished between the op-log read and the fetch
1171                // (erasure, retention) is not an error — it simply no longer
1172                // matches anything.
1173                continue;
1174            };
1175            if !crate::predicate::grain_matches(&grain, &predicate) {
1176                continue;
1177            }
1178            let hex = op.hash.to_hex();
1179            items.push(PollItem {
1180                // The content address IS the identity: the same grain can never
1181                // fire twice, and two evaluators seeing it agree without
1182                // coordinating.
1183                id: hex.clone(),
1184                payload: serde_json::json!({
1185                    "hash": hex,
1186                    "grain_type": grain.grain_type.as_str(),
1187                    "op_seq": op.op_seq,
1188                }),
1189                blobs: Vec::new(),
1190            });
1191        }
1192        Ok((items, cursor))
1193    }
1194
1195    /// Store an item's connector blobs in the CAS and rewrite payload
1196    /// references (#93). Returns one `ContentRef` list per item, parallel
1197    /// to `items`. Any contract violation — undecodable base64, a budget
1198    /// overrun, a dangling `"@N"` — refuses the WHOLE poll with TRG-E011,
1199    /// so the cursor stays put and nothing is silently dropped. `put_blob`
1200    /// is idempotent on content, so a re-polled page costs only the
1201    /// transfer.
1202    fn ingest_item_blobs(
1203        &self,
1204        hash: &str,
1205        items: &mut [PollItem],
1206        opts: &EvalOptions,
1207    ) -> Result<Vec<Vec<ContentRef>>> {
1208        let violation = |detail: String| TriggerError::BlobContract {
1209            trigger: short(hash, 12).to_string(),
1210            detail,
1211        };
1212        let mut response_total = 0usize;
1213        let mut all_refs = Vec::with_capacity(items.len());
1214        for (idx, item) in items.iter_mut().enumerate() {
1215            if item.blobs.is_empty() {
1216                all_refs.push(Vec::new());
1217                continue;
1218            }
1219            let mut item_total = 0usize;
1220            let mut refs = Vec::with_capacity(item.blobs.len());
1221            let mut uris = Vec::with_capacity(item.blobs.len());
1222            for (bi, blob) in item.blobs.iter().enumerate() {
1223                let name = blob.filename.as_deref().unwrap_or("unnamed");
1224                let bytes = areev_core::b64::decode(&blob.b64).ok_or_else(|| {
1225                    violation(format!("item {idx} blob {bi} ({name}) is not valid base64"))
1226                })?;
1227                item_total += bytes.len();
1228                response_total += bytes.len();
1229                if item_total > opts.max_blob_bytes_per_item {
1230                    return Err(violation(format!(
1231                        "item {idx} blobs exceed the per-item budget ({} bytes decoded)",
1232                        opts.max_blob_bytes_per_item
1233                    )));
1234                }
1235                if response_total > opts.max_blob_bytes_per_response {
1236                    return Err(violation(format!(
1237                        "response blobs exceed the per-response budget ({} bytes decoded)",
1238                        opts.max_blob_bytes_per_response
1239                    )));
1240                }
1241                let uri = self
1242                    .facade
1243                    .with_store(|m| m.put_blob(&bytes))
1244                    .map_err(|e| TriggerError::Storage { detail: e.to_string() })?;
1245                refs.push(ContentRef {
1246                    uri: uri.clone(),
1247                    modality: None,
1248                    mime_type: blob.mime.clone(),
1249                    size_bytes: Some(bytes.len() as u64),
1250                    checksum: Some(uri.trim_start_matches("cas://").to_string()),
1251                    metadata: blob.filename.as_ref().map(|f| serde_json::json!({ "filename": f })),
1252                });
1253                uris.push(uri);
1254            }
1255            rewrite_blob_refs(&mut item.payload, &uris)
1256                .map_err(|d| violation(format!("item {idx}: {d}")))?;
1257            // The bytes are in the CAS now; drop the base64 so nothing
1258            // downstream can re-inline it into the Event or the run input.
1259            item.blobs.clear();
1260            all_refs.push(refs);
1261        }
1262        Ok(all_refs)
1263    }
1264
1265    /// Resolve a trigger's `connector_tool` into something executable, or
1266    /// `None` when the trigger names no connector grain and the host command
1267    /// polls as before.
1268    ///
1269    /// Every refusal here happens before the poll has spent anything, and each
1270    /// one names the flag that fixes it — a heartbeat is the surface where a
1271    /// vague failure costs the most, because nobody is reading it.
1272    fn prepare_connector(
1273        &self,
1274        hash: &str,
1275        trigger: &Trigger,
1276        connector_name: &str,
1277    ) -> Result<Option<PreparedConnector>> {
1278        let Some(reference) = trigger
1279            .connector_tool
1280            .as_deref()
1281            .map(str::trim)
1282            .filter(|r| !r.is_empty())
1283        else {
1284            return Ok(None);
1285        };
1286        let refuse = |detail: String| TriggerError::ConnectorCode {
1287            trigger: hash.to_string(),
1288            detail,
1289        };
1290        let Some(cfg) = self.connector_code.as_ref() else {
1291            return Err(refuse(format!(
1292                "connector {connector_name:?} is a grain ({reference}) but this host runs no \
1293                 code-carrying connectors — pass --allow-executor <address> (and --sandbox-cmd \
1294                 for a wasm module)"
1295            )));
1296        };
1297        let hex = areev_core::types::strip_grain_scheme(reference);
1298        let h = Hash::from_hex(hex).map_err(|e| {
1299            refuse(format!("connector_tool {reference:?} is not a content address: {e}"))
1300        })?;
1301        let g = self
1302            .facade
1303            .with_store(|m| m.get(&h))
1304            .map_err(|e| refuse(format!("connector Definition {hex} is unreadable: {e}")))?;
1305        if g.grain_type != GrainType::Tool {
1306            return Err(refuse(format!(
1307                "connector_tool {hex} is a {:?} grain, not a Tool Definition",
1308                g.grain_type
1309            )));
1310        }
1311        // ONE reader of a Definition's executable shape, shared with
1312        // `RunManifest::resolve`: a connector that ran under different rules
1313        // than a node binding the same grain would be a second implementation
1314        // of the pin, the runtime table and the capability check — and the one
1315        // that drifts is this one, because nobody is watching a poll.
1316        let pin = areev_run::pin_from_definition(connector_name, &h, &g)
1317            .map_err(|e| refuse(e.to_string()))?;
1318        let Some(uri) = pin.executor_uri.clone() else {
1319            return Err(refuse(format!(
1320                "connector Definition {hex} names no executor_uri — a connector grain IS its \
1321                 code, so there is nothing to run"
1322            )));
1323        };
1324        // The host pin, checked here so the refusal can name the address to
1325        // copy. `execute_code` checks it again on the way in; that one is the
1326        // guarantee, this one is the error message.
1327        let addr = normalize_address(&uri);
1328        if !cfg.allow.iter().any(|a| normalize_address(a) == addr) {
1329            return Err(refuse(format!(
1330                "connector {connector_name:?} names {uri}, which this host did not pin — \
1331                 add --allow-executor {addr}"
1332            )));
1333        }
1334        if pin.runtime.as_deref().is_some_and(areev_run::is_sandbox_runtime)
1335            && cfg.sandbox_cmd.is_none()
1336        {
1337            return Err(refuse(format!(
1338                "connector {connector_name:?} declares runtime {:?}, which needs the sandbox — \
1339                 pass --sandbox-cmd areev-sandbox",
1340                pin.runtime.as_deref().unwrap_or_default()
1341            )));
1342        }
1343        // `get_blob` verifies the digest on every read, so these bytes ARE the
1344        // address the trigger named.
1345        let bytes = self
1346            .facade
1347            .with_store(|m| m.get_blob(&uri))
1348            .map_err(|e| refuse(format!("connector blob {uri} is unreadable: {e}")))?;
1349        let wants_blobs = pin
1350            .capabilities
1351            .as_ref()
1352            .and_then(|v| areev_run::Declaration::parse(v).ok())
1353            .is_some_and(|d| d.declares_blob_read());
1354        let config = g.fields.get("config").filter(|v| !v.is_null()).cloned();
1355        Ok(Some(PreparedConnector {
1356            tool_hash: h.to_hex(),
1357            code: areev_run::PreparedCode {
1358                uri,
1359                bytes,
1360                runtime: pin.runtime,
1361                limits: pin.runtime_limits,
1362                capabilities: pin.capabilities,
1363            },
1364            wants_blobs,
1365            config,
1366        }))
1367    }
1368
1369    /// The executor a grain connector runs under, bound to THIS poll's broker.
1370    ///
1371    /// Built per poll rather than once on the evaluator because the broker is
1372    /// per poll: it is started with the trigger's own allowlist and stops when
1373    /// the call returns, and a handle to a broker that has stopped would hand
1374    /// the module a dead token.
1375    fn code_executor(&self, broker: &Arc<areev_run::Broker>) -> areev_run::CodeExecutor {
1376        let cfg = self.connector_code.clone().unwrap_or_default();
1377        let mut ce = areev_run::CodeExecutor::new(Arc::new(NoConnectorFallback));
1378        for addr in &cfg.allow {
1379            ce = ce.allow(addr);
1380        }
1381        if let Some(dir) = cfg.cache_dir {
1382            ce = ce.cache_dir(dir);
1383        }
1384        if let Some(cmd) = cfg.sandbox_cmd.as_deref() {
1385            ce = ce.sandbox_cmd(cmd);
1386        }
1387        if let Some(t) = cfg.timeout {
1388            ce = ce.with_timeout(t);
1389        }
1390        if let Some(env) = cfg.env {
1391            ce = ce.with_env_policy(env);
1392        }
1393        ce.with_egress(areev_run::EgressHandle::new(Arc::clone(broker)))
1394    }
1395
1396    fn poll(
1397        &self,
1398        hash: &str,
1399        trigger: &Trigger,
1400        state: &TriggerState,
1401        opts: &EvalOptions,
1402    ) -> Result<PollResponse> {
1403        let connector_name = trigger.connector.as_deref().unwrap_or_default();
1404        // A connector named by content address (#185) is resolved, pinned and
1405        // read BEFORE a broker starts: a Definition this host did not pin must
1406        // refuse having spent nothing — no port, no token, no upstream call.
1407        let prepared = self.prepare_connector(hash, trigger, connector_name)?;
1408        if prepared.is_none() && self.connector.is_none() {
1409            return Err(TriggerError::NoConnector {
1410                trigger: hash.to_string(),
1411                connector: connector_name.to_string(),
1412            });
1413        }
1414        // Start a broker for this call, scoped to this trigger's allowlist. The
1415        // connector is handed its URL — never a token — so a compromised
1416        // connector has nothing to exfiltrate and nowhere undeclared to send
1417        // it. The broker stops when this call returns.
1418        let policy = areev_run::EgressPolicy::from_config(trigger.config.as_ref())
1419            .map_err(|what| TriggerError::Malformed { what })?;
1420        // One connector runs per pass, so there is no second caller to tell it
1421        // apart from: a default grant covers it. Its methods are whatever the
1422        // connector needs to poll, and the credentials are the ones this host
1423        // configured — naming one it was not given still fails.
1424        let grant = self.credentials.keys().fold(
1425            areev_run::CallerGrant::new()
1426                .method("GET")
1427                .method("POST")
1428                .method("PUT")
1429                .method("PATCH")
1430                .method("DELETE"),
1431            |g, name| g.credential(name),
1432        );
1433        // The same grant twice, under two names, because the broker
1434        // identifies a caller by the TOKEN it presents: the default token maps
1435        // to no caller at all, and a capability declaration is registered
1436        // under a name (#185). A grain connector declares
1437        // `{"blob": {"read": true}}` as tool `<connector>`, so it needs a
1438        // token that resolves back to `<connector>` — otherwise its own
1439        // declaration is invisible to the check that reads it, and every blob
1440        // read is refused as undeclared. The default stays for the subprocess
1441        // connector, which presents a token and names nothing.
1442        let grants = areev_run::EgressGrants::new()
1443            .default_for_all(grant.clone())
1444            .grant(connector_name, grant);
1445        let broker = Arc::new(
1446            areev_run::Broker::start(policy, self.credentials.clone(), grants, "TRG-E009")
1447                .map_err(|detail| TriggerError::Storage { detail })?,
1448        );
1449        // A connector module that declared `{"blob": {"read": true}}` reads
1450        // the memory's stored bytes through this same broker on the same
1451        // token (#106) — the attachment an earlier poll filed, by address.
1452        // The read never opens the memory, which is what makes it safe while
1453        // the evaluator holds it.
1454        if prepared.as_ref().is_some_and(|p| p.wants_blobs) {
1455            let locator = self
1456                .connector_code
1457                .as_ref()
1458                .and_then(|c| c.db_locator.as_deref())
1459                .ok_or_else(|| TriggerError::ConnectorCode {
1460                    trigger: hash.to_string(),
1461                    detail: format!(
1462                        "connector {connector_name:?} declares {{\"blob\": {{\"read\": true}}}} \
1463                         but this host wired the evaluator no memory locator, so the broker \
1464                         cannot serve a blob read"
1465                    ),
1466                })?;
1467            broker.serve_blobs(locator);
1468        }
1469
1470        let config = merged_config(
1471            prepared.as_ref().and_then(|p| p.config.as_ref()),
1472            trigger.config.as_ref(),
1473        );
1474        let request = PollRequest {
1475            trigger: hash,
1476            connector: connector_name,
1477            scope: trigger.scope.as_deref(),
1478            cursor: state.cursor.as_deref(),
1479            max_items: opts.max_items,
1480            config: config.as_ref(),
1481        };
1482        let payload = serde_json::to_value(&request).map_err(|e| TriggerError::ConnectorFailed {
1483            trigger: hash.to_string(),
1484            detail: format!("serialize request: {e}"),
1485        })?;
1486        // The idempotency key ties this call to this occurrence, so a connector
1487        // that deduplicates on it sees a retry as a retry.
1488        let idem = format!("{hash}:{}", state.fence);
1489        static EGRESS_ENV: std::sync::Mutex<()> = std::sync::Mutex::new(());
1490        let result = match &prepared {
1491            // The connector is a grain: the pinned blob runs under the same
1492            // `CodeExecutor` the run path uses, so the sandbox argv, the
1493            // capability registration, the per-call broker handshake and the
1494            // cleared environment are ONE implementation rather than a second
1495            // one that drifts. It takes its broker token per call rather than
1496            // through the process environment, so the mutex below is not its
1497            // hazard to hold.
1498            Some(p) => self.code_executor(&broker).execute_code(
1499                connector_name,
1500                &p.tool_hash,
1501                &p.code,
1502                &payload,
1503                &idem,
1504            ),
1505            // The connector reads AREEV_EGRESS_URL out of its environment. The
1506            // spawn seam scrubs whatever `--passphrase-env` and `--token-env`
1507            // name, so this is the only network affordance it has from us.
1508            //
1509            // The variable is PROCESS-GLOBAL, and `HostToolExecutor::execute`
1510            // has no per-call environment, so two evaluators in one process
1511            // would race: one could hand its broker's URL to the other's
1512            // connector, pointing it at the wrong allowlist. The lock makes
1513            // the set→spawn→unset window exclusive. Production runs one
1514            // evaluator per process, so this serialises nothing that was
1515            // parallel; it closes a hazard that only exists in-process (and
1516            // that our own threaded tests could reach).
1517            None => {
1518                let exec = self.connector.as_ref().expect("checked above");
1519                let _guard = EGRESS_ENV.lock().unwrap_or_else(|e| e.into_inner());
1520                std::env::set_var("AREEV_EGRESS_URL", broker.url());
1521                // The broker authenticates its callers: loopback is not an
1522                // authorization, and without the token every call would 401.
1523                if let Some(t) = broker.token_for(connector_name) {
1524                    std::env::set_var("AREEV_EGRESS_TOKEN", t);
1525                }
1526                let r = exec.execute(connector_name, hash, &payload, &idem);
1527                std::env::remove_var("AREEV_EGRESS_URL");
1528                std::env::remove_var("AREEV_EGRESS_TOKEN");
1529                r
1530            }
1531        };
1532
1533        // Surface a refused destination as its own error rather than letting it
1534        // read as a generic connector failure — a connector reaching somewhere
1535        // undeclared is a different problem from one that crashed.
1536        let refused = broker.refusals();
1537        if !refused.is_empty() {
1538            return Err(TriggerError::EgressRefused {
1539                trigger: hash.to_string(),
1540                host: refused
1541                    .iter()
1542                    .map(|r| format!("{} ({})", r.destination, r.reason))
1543                    .collect::<Vec<_>>()
1544                    .join(", "),
1545            });
1546        }
1547        match result {
1548            areev_run::ExecResult::Ok(v) => {
1549                // `{"error": …}` is the shape every blessed blob fails in, and
1550                // `PollResponse` would deserialize it into an empty page with
1551                // no cursor — a source that is down, reported as a source with
1552                // nothing new, on every tick, silently. A poll that answered an
1553                // error FAILED: the claim is released, the cursor stays, and
1554                // the backoff applies (#231).
1555                if let Some(detail) = v
1556                    .get("error")
1557                    .and_then(|e| e.as_str())
1558                    .map(str::trim)
1559                    .filter(|e| !e.is_empty())
1560                {
1561                    let code = v.get("code").and_then(|c| c.as_str()).unwrap_or_default();
1562                    let named =
1563                        if code.is_empty() { String::new() } else { format!(" [{code}]") };
1564                    return Err(TriggerError::ConnectorFailed {
1565                        trigger: hash.to_string(),
1566                        detail: format!("the connector reported an error{named}: {detail}"),
1567                    });
1568                }
1569                serde_json::from_value(v).map_err(|e| TriggerError::ConnectorFailed {
1570                    trigger: hash.to_string(),
1571                    detail: format!("response is not a poll response: {e}"),
1572                })
1573            }
1574            areev_run::ExecResult::Err { detail, .. } => {
1575                Err(TriggerError::ConnectorFailed { trigger: hash.to_string(), detail })
1576            }
1577        }
1578    }
1579
1580    /// Record the item as an Event and start the bound workflow.
1581    ///
1582    /// `legacy_run_ids` (#128) covers a trigger that was superseded BEFORE
1583    /// this fix shipped, when the run id was minted from the (then-current)
1584    /// head rather than the chain root: each entry is the id this same item
1585    /// would have gotten under an older head. Empty for a never-superseded
1586    /// trigger, so it costs nothing in the common case.
1587    fn start_run(
1588        &self,
1589        trigger: &Trigger,
1590        run_id: &str,
1591        legacy_run_ids: &[String],
1592        item: &PollItem,
1593        trigger_hash: &str,
1594        content_refs: &[ContentRef],
1595    ) -> StartOutcome {
1596        // Idempotency has to hold whether or not a runtime is wired in. With a
1597        // starter, the runtime's duplicate-run-id refusal is the guard; without
1598        // one, nothing else would stop the same item being re-ingested on every
1599        // poll forever, because an Event's `created_at` differs per firing and
1600        // so content addressing does not collapse them.
1601        let seen = self
1602            .facade
1603            .with_store(|m| m.run_grains(&self.ns, run_id, 0, 1))
1604            .map(|g| !g.is_empty())
1605            .unwrap_or(false);
1606        if seen {
1607            return StartOutcome::Duplicate;
1608        }
1609        // #128 compat — an item already processed under a pre-fix
1610        // (head-keyed) run id must still be recognized, or a trigger
1611        // superseded before this fix reprocesses its whole backlog the
1612        // first time it is evaluated after upgrading.
1613        for legacy_id in legacy_run_ids {
1614            let seen_legacy = self
1615                .facade
1616                .with_store(|m| m.run_grains(&self.ns, legacy_id, 0, 1))
1617                .map(|g| !g.is_empty())
1618                .unwrap_or(false);
1619            if seen_legacy {
1620                return StartOutcome::Duplicate;
1621            }
1622        }
1623
1624        // The ingest Event is built here but — #129 — WRITTEN only once the
1625        // outcome is one this dedup check should treat as "seen": Ingested
1626        // (no starter, so the Event itself IS the completed unit of work),
1627        // Started, or a starter-reported Duplicate (the runtime already has
1628        // it covered, possibly from elsewhere). A starter-reported `Failed`
1629        // writes NOTHING, so a retry's `seen` check above finds nothing and
1630        // genuinely retries `starter.start()` — before this, the Event was
1631        // always written first, so a failed start still left a `run_id`-
1632        // bearing grain behind that made every future retry of the SAME
1633        // item look like an already-processed duplicate, permanently
1634        // stranding it despite #129's cursor hold.
1635        let build_event = || {
1636            let mut event = Event::new(&item.payload.to_string())
1637                .namespace(&self.ns)
1638                .extra_field("trigger", serde_json::json!(trigger_hash))
1639                .extra_field("run_id", serde_json::json!(run_id));
1640            if let Some(c) = &trigger.connector {
1641                event = event.extra_field("connector", serde_json::json!(c));
1642            }
1643            // #93 — the item's stored blobs, referenced the same way a
1644            // host-driven ingest references them: through `content_refs`,
1645            // which is what keeps them alive through GC, carries them in
1646            // bundles, and lets erasure's sole-reference reclamation find
1647            // them.
1648            for cr in content_refs {
1649                event = event.content_ref(cr.clone());
1650            }
1651            event
1652        };
1653
1654        let Some(starter) = &self.starter else {
1655            // Ingest-only: the item is recorded, nothing is executed. Reported
1656            // as ingested rather than as a run, because claiming a run started
1657            // when none did would make the report lie.
1658            return match self.facade.with_store(|m| m.add(&build_event())) {
1659                Ok(_) => StartOutcome::Ingested,
1660                Err(e) => StartOutcome::Failed(format!("ingest: {e}")),
1661            };
1662        };
1663        let mut input = serde_json::json!({
1664            "trigger": trigger_hash,
1665            "connector": trigger.connector,
1666            "scope": trigger.scope,
1667            "item": item.payload,
1668        });
1669        // Declared context (#85): the EVALUATOR assembles it, because it is
1670        // the one party that already holds the memory — on the embedded
1671        // backend a tool inside the run cannot open the file its own run
1672        // locks. Fail closed: a trigger that declared context must not fire
1673        // blind, so a missing query or a failed read refuses the firing (the
1674        // evaluator's normal retry/backoff applies) rather than starting a
1675        // run without the context its declaration promised.
1676        if let Some(spec) = &trigger.context_query {
1677            match self.assemble_context(spec, &item.payload) {
1678                Ok(ctx) => {
1679                    input["context"] = ctx;
1680                }
1681                Err(why) => {
1682                    return StartOutcome::Failed(format!("context_query \"{spec}\": {why}"));
1683                }
1684            }
1685        }
1686        // Read through whatever scheme prefix the declaration spelled the
1687        // reference with (#73): `sha256:<hex>` validated, listed, and reported
1688        // `waiting` forever, then died here on `FMT-E001: invalid hex hash`.
1689        match starter.start(trigger.workflow_hash(), run_id, input) {
1690            // The run is durably underway (or already was, started elsewhere)
1691            // before we ever try to write the ingest Event, so a failure to
1692            // write it here does not need to be reported as a failed firing:
1693            // a retry would find `starter.start()` itself now reporting
1694            // `Duplicate` (the runtime's own dedup), never a second run. Best
1695            // effort is enough; the Event is audit/context linkage, not the
1696            // run's own record of itself.
1697            StartResult::Started => {
1698                let _ = self.facade.with_store(|m| m.add(&build_event()));
1699                StartOutcome::Started
1700            }
1701            StartResult::Duplicate => {
1702                let _ = self.facade.with_store(|m| m.add(&build_event()));
1703                StartOutcome::Duplicate
1704            }
1705            StartResult::Failed(why) => StartOutcome::Failed(why),
1706        }
1707    }
1708
1709    /// Run the trigger's saved query (`RUN "name"($p = v, …)`) against the
1710    /// facade this evaluator already holds and return the result payload.
1711    /// The executor re-validates the saved body as read-only at execution,
1712    /// and the destructive cap is off — a context query can never mutate.
1713    ///
1714    /// #92 — the declaration may bind the query's parameters from the
1715    /// firing item (`name($session = /session)`): each JSON pointer is
1716    /// resolved against the item's payload with `--dedup-key`'s machinery
1717    /// and semantics. Fail-closed like the rest of declared context: an
1718    /// unresolvable pointer, or one landing on a non-scalar, refuses the
1719    /// firing rather than running the query with a hole in it. Values ride
1720    /// as a parsed AST (`RunQueryStmt`), never inside CAL text, so an
1721    /// untrusted payload value cannot inject CAL.
1722    fn assemble_context(
1723        &self,
1724        spec: &str,
1725        payload: &serde_json::Value,
1726    ) -> std::result::Result<serde_json::Value, String> {
1727        let spec = crate::context::parse_context_query(spec)?;
1728        let mut bindings: Vec<(String, areev_cal::ast::Value)> = Vec::new();
1729        for (param, pointer) in &spec.bindings {
1730            let v = payload.pointer(pointer).ok_or_else(|| {
1731                format!("parameter ${param}: pointer {pointer} does not resolve against the firing item")
1732            })?;
1733            let value = match v {
1734                serde_json::Value::String(s) => {
1735                    areev_cal::ast::Value::String { value: s.clone() }
1736                }
1737                serde_json::Value::Number(n) => areev_cal::ast::Value::Number {
1738                    value: n.as_f64().ok_or_else(|| {
1739                        format!("parameter ${param}: {pointer} is not a representable number")
1740                    })?,
1741                },
1742                serde_json::Value::Bool(b) => areev_cal::ast::Value::Boolean { value: *b },
1743                other => {
1744                    return Err(format!(
1745                        "parameter ${param}: pointer {pointer} must land on a scalar, found {}",
1746                        match other {
1747                            serde_json::Value::Null => "null",
1748                            serde_json::Value::Array(_) => "an array",
1749                            _ => "an object",
1750                        }
1751                    ))
1752                }
1753            };
1754            bindings.push((param.clone(), value));
1755        }
1756        let ex = areev_cal::CalExecutor::new(areev_cal::CalExecutorConfig {
1757            allow_destructive_ops: false,
1758            ..areev_cal::CalExecutorConfig::default()
1759        });
1760        let query = areev_cal::ast::CalQuery {
1761            version: Default::default(),
1762            statement: areev_cal::ast::CalStatement::RunQuery(areev_cal::ast::RunQueryStmt {
1763                name: spec.name.clone(),
1764                bindings,
1765                span: None,
1766            }),
1767            pipeline: Vec::new(),
1768            with_options: Vec::new(),
1769            format: None,
1770            let_bindings: Vec::new(),
1771            user_vars: Default::default(),
1772            let_values: Default::default(),
1773            warnings: Vec::new(),
1774        };
1775        let res = ex
1776            .execute_parsed(query, &format!("RUN \"{}\"", spec.name), self.facade.as_ref())
1777            .map_err(|e| e.to_string())?;
1778        serde_json::to_value(&res.result).map_err(|e| e.to_string())
1779    }
1780
1781    /// One Observation per firing: what fired, what it produced, what it
1782    /// skipped. An unfired trigger is invisible otherwise.
1783    fn journal(
1784        &self,
1785        hash: &str,
1786        trigger: &Trigger,
1787        outcome: &FireOutcome,
1788        note: Option<&str>,
1789        now: i64,
1790    ) -> Result<()> {
1791        let mut obs = Observation::new("areev:trigger", "trigger:firing")
1792            .namespace(TRIGGER_NS)
1793            .created_at(now)
1794            .extra_field("trigger", serde_json::json!(hash))
1795            .extra_field("kind", serde_json::json!(trigger.kind.as_str()))
1796            .extra_field("workflow", serde_json::json!(trigger.workflow))
1797            .extra_field("items", serde_json::json!(outcome.items))
1798            .extra_field("runs_started", serde_json::json!(outcome.runs_started))
1799            .extra_field("duplicates", serde_json::json!(outcome.duplicates));
1800        // Without these, a firing where every item lacked the dedup key reads
1801        // as "items 5, runs_started 0, duplicates 0" -- and that last zero
1802        // actively misleads, saying the items were not skipped as duplicates
1803        // without saying why they were skipped at all. Emitted only when
1804        // non-zero, so an ordinary firing stays as small as it was.
1805        if outcome.unidentifiable > 0 {
1806            obs = obs.extra_field("unidentifiable", serde_json::json!(outcome.unidentifiable));
1807        }
1808        if outcome.ingested > 0 {
1809            obs = obs.extra_field("ingested", serde_json::json!(outcome.ingested));
1810        }
1811        if !outcome.failures.is_empty() {
1812            obs = obs.extra_field("failures", serde_json::json!(outcome.failures));
1813            // #129 — names WHY the cursor did not move this firing, so the
1814            // audit record does not require the reader to infer it from
1815            // `duplicates` staying flat on the next firing.
1816            obs = obs.extra_field("cursor_held", serde_json::json!(true));
1817        }
1818        if outcome.seeded {
1819            obs = obs.extra_field("seeded", serde_json::json!(true));
1820        }
1821        if let Some(n) = note {
1822            obs = obs.extra_field("note", serde_json::json!(n));
1823        }
1824        self.facade
1825            .with_store(|m| m.add_if_novel(&obs))
1826            .map(|_| ())
1827            .map_err(|e| TriggerError::Storage { detail: e.to_string() })
1828    }
1829
1830    /// The current head of this trigger's PLAN, when the plan has been edited
1831    /// since the trigger was declared. `None` when the trigger already points
1832    /// at the live version — the overwhelmingly common case, and one forward
1833    /// point read to establish.
1834    ///
1835    /// Deliberately a report and not a redirect. Following the chain at fire
1836    /// time would mean a plan edit silently changes what an unattended
1837    /// heartbeat runs, which is exactly what content-addressed binding exists
1838    /// to prevent; the operator moves the pointer with `trigger retarget` when
1839    /// they mean to.
1840    pub fn plan_head_if_stale(&self, t: &Trigger) -> Result<Option<String>> {
1841        let Ok(h) = Hash::from_hex(t.workflow_hash()) else {
1842            // Not a content address at all — a different complaint, and
1843            // `unusable` is where that one is reported.
1844            return Ok(None);
1845        };
1846        let head = self
1847            .facade
1848            .with_store(|m| m.current_head(&h))
1849            .map_err(|e| TriggerError::Storage { detail: e.to_string() })?;
1850        Ok((head != h).then(|| head.to_hex()))
1851    }
1852
1853    /// Resolve (and cache) the supersession chain for one trigger's head
1854    /// hash within one evaluation pass (#128).
1855    ///
1856    /// `cache` lives on the CALLER's stack — a fresh `HashMap` per `run()`,
1857    /// `deliver()`, or `status()` invocation — so a chain is walked AT MOST
1858    /// ONCE per trigger per pass no matter how many items it fires or how
1859    /// many composites settle against it in that pass. Do not add a call
1860    /// site that walks the chain per item; thread the already-resolved
1861    /// [`Chain`] through instead, the way `fire`/`collect_and_start` do.
1862    fn resolve_chain(&self, hash: &str, cache: &mut HashMap<String, Chain>) -> Result<Chain> {
1863        if let Some(c) = cache.get(hash) {
1864            return Ok(c.clone());
1865        }
1866        let h = Hash::from_hex(hash).map_err(|e| TriggerError::Storage { detail: e.to_string() })?;
1867        let hashes: Vec<String> = self
1868            .facade
1869            .with_store(|m| m.supersession_chain(&h))
1870            .map_err(|e| TriggerError::Storage { detail: e.to_string() })?
1871            .into_iter()
1872            .map(|hh| hh.to_hex())
1873            .collect();
1874        let chain = Chain(hashes);
1875        cache.insert(hash.to_string(), chain.clone());
1876        Ok(chain)
1877    }
1878
1879    /// Read a trigger's evaluation state through its resolved chain, with
1880    /// compatibility for a trigger superseded BEFORE #128 shipped (its state
1881    /// was keyed on the head hash of the moment, not the chain root).
1882    ///
1883    /// Always tries the ROOT key first. For a trigger that has never been
1884    /// superseded this is the ONLY path ever taken — `chain.root() ==` the
1885    /// hash this crate always keyed state on — so the common case is a
1886    /// byte-identical lookup to before this feature existed.
1887    ///
1888    /// If the root row is absent, every other hash in the chain
1889    /// (`chain.legacy()`) is checked for a row written under an older head.
1890    /// Several may exist if the trigger was superseded more than once
1891    /// before upgrading; [`state_is_more_advanced`] picks whichever
1892    /// represents the most real progress, so the survivor is never the one
1893    /// that would silently throw away a further-along cursor.
1894    ///
1895    /// `adopt = true` persists the found legacy state under the root and
1896    /// deletes the legacy row — write the new row FIRST, delete the legacy
1897    /// row SECOND, so a crash in between leaves a harmless duplicate (the
1898    /// legacy row is simply never consulted again once the root row exists)
1899    /// rather than losing the cursor. Pass `true` only from a caller that
1900    /// already owns write access for real evaluation work (`run`,
1901    /// `deliver`, composite settlement); `status`/`show` pass `false` so a
1902    /// pure inspection command has no side effect. Adoption is best-effort
1903    /// against a read-only-opened memory: `STO-E004` is swallowed and the
1904    /// found state is still returned, just not yet persisted — the next
1905    /// pass with write access adopts it.
1906    fn resolve_state(&self, chain: &Chain, adopt: bool) -> Result<(TriggerState, Option<String>)> {
1907        let root = chain.root();
1908        if let Some((state, raw)) = self
1909            .facade
1910            .with_store(|m| m.trigger_state(root))
1911            .map_err(|e| TriggerError::Storage { detail: e.to_string() })?
1912        {
1913            return Ok((state, Some(raw)));
1914        }
1915
1916        let mut best: Option<(&str, TriggerState)> = None;
1917        for legacy_hash in chain.legacy() {
1918            if let Some((state, _raw)) = self
1919                .facade
1920                .with_store(|m| m.trigger_state(legacy_hash))
1921                .map_err(|e| TriggerError::Storage { detail: e.to_string() })?
1922            {
1923                let better = best
1924                    .as_ref()
1925                    .is_none_or(|(_, current)| state_is_more_advanced(&state, current));
1926                if better {
1927                    best = Some((legacy_hash.as_str(), state));
1928                }
1929            }
1930        }
1931        let Some((legacy_hash, state)) = best else {
1932            // No row anywhere in the chain: a genuinely new trigger.
1933            return Ok((TriggerState::default(), None));
1934        };
1935        if !adopt {
1936            return Ok((state, None));
1937        }
1938
1939        // `expected=None` only claims an ABSENT row (`INSERT OR IGNORE`), so
1940        // losing a race to another evaluator adopting the same legacy row
1941        // concurrently is a no-op, never a clobber — either way the legacy
1942        // row is redundant now.
1943        match self.facade.with_store(|m| m.put_trigger_state(root, None, &state)) {
1944            Ok(_) => {}
1945            Err(AreevError::ReadOnly(_)) => return Ok((state, None)),
1946            Err(e) => return Err(TriggerError::Storage { detail: e.to_string() }),
1947        }
1948        if let Err(e) = self.facade.with_store(|m| m.clear_trigger_state(legacy_hash)) {
1949            if !matches!(e, AreevError::ReadOnly(_)) {
1950                return Err(TriggerError::Storage { detail: e.to_string() });
1951            }
1952        }
1953        let (state, raw) = self
1954            .facade
1955            .with_store(|m| m.trigger_state(root))
1956            .map_err(|e| TriggerError::Storage { detail: e.to_string() })?
1957            .ok_or_else(|| TriggerError::Storage {
1958                detail: format!(
1959                    "adopted trigger state for {root} vanished immediately after writing"
1960                ),
1961            })?;
1962        Ok((state, Some(raw)))
1963    }
1964}
1965
1966/// A trigger's supersession chain for the duration of one evaluation pass:
1967/// every content-address hash in its edit history, HEAD (the declaration's
1968/// current hash — what the caller resolved from) first, ROOT last (#128).
1969///
1970/// Evaluation state (`trg:<hash>`) and derived run ids are keyed on the
1971/// ROOT, never the head: a Trigger grain re-pointed by `SUPERSEDE` mints a
1972/// new head hash for the SAME standing rule (`docs/loop.md`'s
1973/// propose→approve→apply cycle re-points a trigger exactly this way), and
1974/// keying on the head alone would orphan the cursor and the dedup fence
1975/// every time a recommendation is applied. For a trigger that has never
1976/// been superseded, `root() == head`, so every key derived from it is
1977/// byte-identical to what this crate always computed — the no-op guarantee
1978/// the fix has to preserve.
1979#[derive(Debug, Clone)]
1980struct Chain(Vec<String>);
1981
1982impl Chain {
1983    /// The key evaluation state and run ids are actually resolved against.
1984    fn root(&self) -> &str {
1985        self.0.last().map(String::as_str).expect("a chain always has at least its head")
1986    }
1987
1988    /// Every hash besides the root — the identities a pre-#128 evaluator
1989    /// might have keyed state or a run id under. Empty for a
1990    /// never-superseded trigger, so the compatibility paths that walk this
1991    /// cost nothing in the common case.
1992    fn legacy(&self) -> &[String] {
1993        &self.0[..self.0.len() - 1]
1994    }
1995}
1996
1997/// Which of two legacy state rows found along a chain is "further along"
1998/// and should be adopted, when more than one exists (a trigger superseded
1999/// more than once before #128 shipped). `last_fired_at` is the most direct
2000/// evidence of real progress across every trigger kind — the row that
2001/// fired most recently is the row whose cursor best reflects what has
2002/// actually been processed. `op_cursor` (a `memory` trigger's position) and
2003/// then "has this row ever polled at all" break a tie where neither
2004/// candidate has ever fired.
2005fn state_is_more_advanced(candidate: &TriggerState, current_best: &TriggerState) -> bool {
2006    (candidate.last_fired_at, candidate.op_cursor, candidate.cursor.is_some())
2007        > (current_best.last_fired_at, current_best.op_cursor, current_best.cursor.is_some())
2008}
2009
2010/// One line for `released.last_error` (#129): several items in one firing
2011/// can fail for different reasons, and `last_error` is a single string.
2012fn summarize_failures(failures: &[String]) -> String {
2013    match failures {
2014        [one] => one.clone(),
2015        many => format!("{} item(s) failed to start: {}", many.len(), many.join("; ")),
2016    }
2017}
2018
2019/// Drop partial matches past their window.
2020///
2021/// Without this, a composite that never completes accumulates keys forever and
2022/// a stale half-match eventually pairs with unrelated work — the exact failure
2023/// Argo Events' reset cron exists to paper over.
2024fn prune_partials(state: &mut TriggerState, trigger: &Trigger, now: i64) {
2025    let Some(window) = trigger.window_ms else {
2026        return;
2027    };
2028    state.partials.retain(|_, p| now - p.started_ms <= window);
2029}
2030
2031#[derive(Debug, Default, Clone)]
2032struct FireOutcome {
2033    items: usize,
2034    runs_started: usize,
2035    ingested: usize,
2036    duplicates: usize,
2037    unidentifiable: usize,
2038    failures: Vec<String>,
2039    cursor: Option<String>,
2040    /// The payloads this firing produced, so a composite can pull its
2041    /// correlation value out of them.
2042    fired_items: Vec<serde_json::Value>,
2043    /// Correlation keys a composite consumed, cleared on release.
2044    settled_keys: Vec<String>,
2045    op_cursor: Option<i64>,
2046    draining: bool,
2047    seeded: bool,
2048    skipped_locked: bool,
2049}
2050
2051enum StartOutcome {
2052    Started,
2053    /// Recorded but not executed — no runtime is wired in.
2054    Ingested,
2055    Duplicate,
2056    Failed(String),
2057}
2058
2059/// The run id for one firing of one item.
2060///
2061/// Derived from `(trigger, connector, dedup value)`, so the same item seen
2062/// twice — connector replay, overlapping cursors, two nodes racing a lease
2063/// boundary — produces the same id. `areev run start` refuses a duplicate, so
2064/// the second attempt is a recorded skip rather than a second run. At-most-once
2065/// firing with no lease, no fencing token, and no clock assumption.
2066///
2067/// The connector is part of the identity because CloudEvents made that
2068/// normative for the same reason: `source` + `id` is unique, `id` alone is only
2069/// unique within a producer. Two connectors sharing an id space must not
2070/// collide.
2071///
2072/// **`trigger_hash` is the chain ROOT (#128), not necessarily the
2073/// declaration's current head.** A `Trigger` grain re-pointed by
2074/// `SUPERSEDE` mints a new head for the same standing rule, and this
2075/// function has no way to know that on its own — a caller that passed the
2076/// head would mint a fresh id for an item already processed under the old
2077/// head, defeating the very dedup fence this function exists to provide.
2078/// Every call site resolves the root once per pass (`Evaluator::
2079/// resolve_chain`) and passes that; this function's signature and hashing
2080/// are otherwise unchanged.
2081pub fn run_id_for(trigger_hash: &str, connector: Option<&str>, dedup_value: &str) -> String {
2082    let mut h = Sha256::new();
2083    h.update(b"areev-trigger:v1");
2084    h.update([0x1f]);
2085    h.update(connector.unwrap_or_default().as_bytes());
2086    h.update([0x1f]);
2087    h.update(dedup_value.as_bytes());
2088    let digest = hex(&h.finalize());
2089    format!("{}-{}", short(trigger_hash, 8), short(&digest, 12))
2090}
2091
2092fn hex(bytes: &[u8]) -> String {
2093    bytes.iter().map(|b| format!("{b:02x}")).collect()
2094}
2095
2096#[cfg(test)]
2097mod tests {
2098    use super::*;
2099
2100    #[test]
2101    fn a_run_id_is_stable_for_the_same_item() {
2102        let a = run_id_for("abcdef1234", Some("gmail"), "m1");
2103        let b = run_id_for("abcdef1234", Some("gmail"), "m1");
2104        assert_eq!(a, b, "the same item must always mint the same id");
2105    }
2106
2107    #[test]
2108    fn different_items_get_different_ids() {
2109        let a = run_id_for("abcdef1234", Some("gmail"), "m1");
2110        let b = run_id_for("abcdef1234", Some("gmail"), "m2");
2111        assert_ne!(a, b);
2112    }
2113
2114    #[test]
2115    fn the_connector_is_part_of_the_identity() {
2116        // CloudEvents' rule: `id` alone is unique only within a producer, so two
2117        // connectors with a shared id space must not collide.
2118        let a = run_id_for("abcdef1234", Some("gmail"), "1");
2119        let b = run_id_for("abcdef1234", Some("stripe"), "1");
2120        assert_ne!(a, b);
2121    }
2122
2123    #[test]
2124    fn the_id_names_its_trigger_for_a_human_reading_a_run_list() {
2125        let id = run_id_for("abcdef1234567890", Some("gmail"), "m1");
2126        assert!(id.starts_with("abcdef12"), "{id}");
2127        assert!(id.len() < 32, "short enough to read: {id}");
2128    }
2129
2130    #[test]
2131    fn a_short_trigger_hash_does_not_panic() {
2132        let id = run_id_for("ab", None, "x");
2133        assert!(id.starts_with("ab-"));
2134    }
2135}