Skip to main content

kernel_apis/
schedules.rs

1//! `schedules.*` and `goals.*` (port of `ultron/schedules.ts`): recurring task firings and goals with required
2//! checks. Firings are admitted through [`TaskHost::spawn`] with a slot-derived key (`schedule:<id>:<slot>`), and
3//! each slot is claimed durably (`last_slot`, `pending_slot`) before admission, so duplicate ticks and restarts never
4//! run a slot twice; a claimed slot that fails is recorded once and never retried. Missed slots coalesce into one run
5//! of the latest slot (`skipped`). A goal is achieved only when every required check succeeds with `passed: true`;
6//! `max_tasks` caps firings, verification and attachments. The document is `{version: 1, schedules, goals}`.
7//!
8//! The TS module ticks on a background interval timer (1 s). Here [`crate::schedule_tick::ScheduleTicker`] sleeps
9//! until the earliest due slot ([`SchedulesModule::next_fire`], derived from each schedule's durable `start_at`,
10//! `every_ms`, `last_slot` and `pending_slot`, so a reopened module recomputes it) and is woken by every committed
11//! change; the model may still send `schedules.tick`.
12
13use crate::store::read_versioned;
14use crate::tasks::TaskHost;
15use crate::util::Clock;
16use kernel::store::DocumentStore;
17use serde_json::{Map, Value, json};
18use std::collections::HashSet;
19use std::sync::{Arc, Mutex};
20
21const MAX_EVERY_MS: i64 = 366 * 24 * 60 * 60 * 1000;
22
23fn definition_key(value: Option<&Value>, name: &str) -> Result<String, String> {
24    match value.and_then(Value::as_str) {
25        Some(text) if crate::util::valid_definition(text) => Ok(text.to_string()),
26        _ => Err(format!("{name} must be id@version")),
27    }
28}
29
30fn integer(value: Option<&Value>, name: &str, min: i64, max: i64) -> Result<i64, String> {
31    value
32        .filter(|value| value.is_number())
33        .and_then(jsrt::number::safe_integer)
34        .filter(|number| (min..=max).contains(number))
35        .ok_or_else(|| format!("{name} must be an integer between {min} and {max}"))
36}
37
38fn optional_integer(value: Option<&Value>, name: &str, min: i64) -> Result<Option<i64>, String> {
39    match value {
40        None | Some(Value::Null) => Ok(None),
41        value => integer(value, name, min, jsrt::number::MAX_SAFE_INTEGER).map(Some),
42    }
43}
44
45fn validate_document(saved: Value) -> Result<Value, String> {
46    let invalid = || "Invalid schedule document".to_string();
47    let map = saved.as_object().ok_or_else(invalid)?;
48    let (Some(schedules), Some(goals)) = (
49        map.get("schedules").and_then(Value::as_array),
50        map.get("goals").and_then(Value::as_array),
51    ) else {
52        return Err(invalid());
53    };
54    if map.get("version") != Some(&json!(1)) {
55        return Err(invalid());
56    }
57    let mut ids = HashSet::new();
58    for item in schedules.iter().chain(goals) {
59        let id = item
60            .as_object()
61            .and_then(|item| item.get("id"))
62            .and_then(Value::as_str)
63            .ok_or_else(invalid)?;
64        if !ids.insert(id.to_string()) {
65            return Err(invalid());
66        }
67    }
68    Ok(saved)
69}
70
71/// Slot index for `now`; negative before the schedule starts.
72pub fn schedule_slot(start_at: i64, every_ms: i64, now: i64) -> i64 {
73    (now - start_at).div_euclid(every_ms)
74}
75
76fn schedule_key(schedule: &Value, slot: i64) -> String {
77    format!("schedule:{}:{slot}", schedule["id"].as_str().unwrap_or(""))
78}
79
80fn check_outcome(definition: &str, task_id: Option<&str>, result: &Value) -> Value {
81    let status = result["status"].clone();
82    let mut outcome = json!({"definition": definition, "task_id": task_id, "status": status, "passed": false});
83    if result["status"] != "succeeded" {
84        let error = match result.get("error").and_then(Value::as_str).filter(|e| !e.is_empty()) {
85            Some(error) => format!(": {error}"),
86            None => String::new(),
87        };
88        outcome["reason"] = format!("{definition} {}{error}", result["status"].as_str().unwrap_or("")).into();
89        return outcome;
90    }
91    match result["value"]
92        .get("passed")
93        .and_then(Value::as_bool)
94        .filter(|_| result["value"].is_object())
95    {
96        None => outcome["reason"] = format!("{definition} did not report a boolean passed value").into(),
97        Some(passed) => {
98            outcome["passed"] = passed.into();
99            if !passed {
100                outcome["reason"] = format!("{definition} reported passed=false").into();
101            }
102        }
103    }
104    outcome
105}
106
107fn missing(definition: &str, task_id: Option<&str>, error: &str) -> Value {
108    json!({"definition": definition, "task_id": task_id, "status": "missing", "passed": false,
109        "reason": format!("{definition}: {error}")})
110}
111
112fn find<'a>(items: &'a mut Value, key: &str, id: &str) -> Option<&'a mut Value> {
113    items[key].as_array_mut()?.iter_mut().find(|item| item["id"] == id)
114}
115
116fn lookup(document: &Value, key: &str, id: Option<&Value>, unknown: &str) -> Result<Value, String> {
117    let id = jsrt::payload::nonempty_string(id, "id")?;
118    document[key]
119        .as_array()
120        .and_then(|items| items.iter().find(|item| item["id"] == id.as_str()))
121        .cloned()
122        .ok_or_else(|| unknown.to_string())
123}
124
125/// Remaining task slots of a goal (`None`: unlimited).
126fn capacity(goal: &Value) -> Option<i64> {
127    let used = goal["task_ids"].as_array().map_or(0, Vec::len) as i64;
128    goal["max_tasks"].as_i64().map(|max| max - used)
129}
130
131fn has_capacity(goal: &Value, needed: i64) -> bool {
132    capacity(goal).is_none_or(|left| left >= needed)
133}
134
135fn push_task(goal: &mut Value, task_id: &Value) {
136    if let Some(ids) = goal["task_ids"].as_array_mut()
137        && !ids.contains(task_id)
138    {
139        ids.push(task_id.clone());
140    }
141}
142
143/// When `schedule` next fires: a claimed slot whose admission was not recorded is due at once (`i64::MIN`); else the
144/// start of the slot after `last_slot` (slot 0 at `start_at`). `None` when it does not fire: not active, or its goal is
145/// missing or not active. A time in the past means due now; missed slots then coalesce into one run.
146pub fn next_fire_at(schedule: &Value, goals: &[Value]) -> Option<i64> {
147    if schedule["state"] != "active" {
148        return None;
149    }
150    if let Some(goal_id) = schedule["goal_id"].as_str()
151        && !goals
152            .iter()
153            .any(|goal| goal["id"] == goal_id && goal["state"] == "active")
154    {
155        return None;
156    }
157    if schedule["pending_slot"].is_i64() {
158        return Some(i64::MIN);
159    }
160    let every_ms = schedule["every_ms"].as_i64().filter(|every| *every > 0)?;
161    let next_slot = schedule["last_slot"].as_i64().map_or(0, |last| last.saturating_add(1));
162    Some(
163        schedule["start_at"]
164            .as_i64()
165            .unwrap_or(0)
166            .saturating_add(next_slot.saturating_mul(every_ms)),
167    )
168}
169
170/// Called after every committed change (the ticker re-reads the next fire time).
171pub type ChangeHook = Arc<dyn Fn() + Send + Sync>;
172
173pub struct SchedulesModule {
174    store: Arc<dyn DocumentStore>,
175    on_change: Mutex<Option<ChangeHook>>,
176    /// `None`: the host's clock.
177    clock: Option<Clock>,
178    /// The loaded document; every read-modify-write runs under this lock.
179    document: Mutex<Option<Value>>,
180}
181
182impl SchedulesModule {
183    pub fn new(store: Arc<dyn DocumentStore>, clock: Option<Clock>) -> Self {
184        Self {
185            store,
186            on_change: Mutex::new(None),
187            clock,
188            document: Mutex::new(None),
189        }
190    }
191
192    /// Calls `hook` after every committed change (the ticker's wake-up).
193    pub fn on_change(&self, hook: Option<ChangeHook>) {
194        *jsrt::sync::lock(&self.on_change) = hook;
195    }
196
197    fn changed(&self) {
198        let hook = jsrt::sync::lock(&self.on_change).clone();
199        if let Some(hook) = hook {
200            hook();
201        }
202    }
203
204    /// The earliest time a schedule fires (see [`next_fire_at`]), after the document loads; `None`: none will.
205    pub fn next_fire(&self) -> Result<Option<i64>, String> {
206        let mut guard = jsrt::sync::lock(&self.document);
207        let document = self.loaded(&mut guard)?;
208        let goals = document["goals"].as_array().cloned().unwrap_or_default();
209        Ok(document["schedules"]
210            .as_array()
211            .into_iter()
212            .flatten()
213            .filter_map(|schedule| next_fire_at(schedule, &goals))
214            .min())
215    }
216
217    fn now(&self, host: &dyn TaskHost) -> i64 {
218        self.clock.as_ref().map_or_else(|| host.now(), |clock| clock())
219    }
220
221    fn loaded<'a>(&self, slot: &'a mut Option<Value>) -> Result<&'a mut Value, String> {
222        if slot.is_none() {
223            *slot = Some(match self.store.read()? {
224                None => json!({"version": 1, "schedules": [], "goals": []}),
225                saved => {
226                    validate_document(read_versioned("ultron.module/schedules", saved, "version")?.unwrap_or_default())?
227                }
228            });
229        }
230        Ok(slot.as_mut().expect("loaded"))
231    }
232
233    /// Mutates a copy, commits it, then publishes it; a failed mutation or write leaves the old state.
234    fn update<T>(
235        &self,
236        document: &mut Value,
237        mutate: impl FnOnce(&mut Value) -> Result<T, String>,
238    ) -> Result<T, String> {
239        let mut draft = document.clone();
240        let result = mutate(&mut draft)?;
241        self.store.write(&draft)?;
242        *document = draft;
243        self.changed();
244        Ok(result)
245    }
246
247    /// Recovers goal task links written by keys the host already admitted before a crash.
248    fn reconcile(&self, document: &mut Value, host: &dyn TaskHost) -> Result<(), String> {
249        if document["goals"].as_array().is_none_or(Vec::is_empty) {
250            return Ok(());
251        }
252        let tasks = host.tasks()?;
253        let mut draft = document.clone();
254        let mut changed = false;
255        for goal in draft["goals"].as_array_mut().into_iter().flatten() {
256            let prefix = format!("goal:{}:", goal["id"].as_str().unwrap_or(""));
257            for task in &tasks {
258                if task["key"].as_str().is_some_and(|key| key.starts_with(&prefix))
259                    && !goal["task_ids"].as_array().is_some_and(|ids| ids.contains(&task["id"]))
260                {
261                    push_task(goal, &task["id"]);
262                    changed = true;
263                }
264            }
265        }
266        let schedules = draft["schedules"].as_array().cloned().unwrap_or_default();
267        for schedule in schedules {
268            let Some(goal_id) = schedule["goal_id"].as_str() else {
269                continue;
270            };
271            let slot = schedule["pending_slot"]
272                .as_i64()
273                .or_else(|| schedule["last_slot"].as_i64());
274            let task = slot.and_then(|slot| {
275                let key = schedule_key(&schedule, slot);
276                tasks.iter().find(|task| task["key"] == key.as_str())
277            });
278            if let (Some(goal), Some(task)) = (find(&mut draft, "goals", goal_id), task)
279                && !goal["task_ids"].as_array().is_some_and(|ids| ids.contains(&task["id"]))
280            {
281                push_task(goal, &task["id"]);
282                changed = true;
283            }
284        }
285        if changed {
286            self.store.write(&draft)?;
287            *document = draft;
288        }
289        Ok(())
290    }
291
292    pub fn start(&self, host: &dyn TaskHost) -> Result<(), String> {
293        let mut guard = jsrt::sync::lock(&self.document);
294        let document = self.loaded(&mut guard)?;
295        self.reconcile(document, host)
296    }
297
298    pub fn handle(
299        &self,
300        kind: &str,
301        payload: &Map<String, Value>,
302        caller: Option<&str>,
303        host: &dyn TaskHost,
304    ) -> Result<Value, String> {
305        if kind == "goals.verify" {
306            return self.verify(payload, caller, host);
307        }
308        let mut guard = jsrt::sync::lock(&self.document);
309        let document = self.loaded(&mut guard)?;
310        match kind {
311            "schedules.create" => self.create_schedule(document, payload, caller, host),
312            "schedules.list" => {
313                kernel::ops::fields("schedules.list", payload)?;
314                Ok(document["schedules"].clone())
315            }
316            "schedules.pause" => self.set_schedule_state(document, kind, payload, "paused"),
317            "schedules.resume" => self.set_schedule_state(document, kind, payload, "active"),
318            "schedules.delete" => {
319                kernel::ops::fields("schedules.delete", payload)?;
320                self.update(document, |draft| {
321                    let schedule = lookup(draft, "schedules", payload.get("id"), "Unknown schedule")?;
322                    if let Some(items) = draft["schedules"].as_array_mut() {
323                        items.retain(|item| item["id"] != schedule["id"]);
324                    }
325                    Ok(json!({"deleted": true}))
326                })
327            }
328            "schedules.tick" => {
329                kernel::ops::fields("schedules.tick", payload)?;
330                // Firings outlive the request that happened to drive the tick.
331                self.tick_locked(document, host)
332            }
333            "goals.create" => self.create_goal(document, payload, caller, host),
334            "goals.list" => {
335                kernel::ops::fields("goals.list", payload)?;
336                Ok(document["goals"].clone())
337            }
338            "goals.get" => {
339                kernel::ops::fields("goals.get", payload)?;
340                lookup(document, "goals", payload.get("id"), "Unknown goal")
341            }
342            "goals.pause" => self.set_goal_state(document, kind, payload, "paused"),
343            "goals.resume" => self.set_goal_state(document, kind, payload, "active"),
344            "goals.attach" => self.attach(document, payload, host),
345            _ => Err(format!("Unknown schedule request: {kind}")),
346        }
347    }
348
349    fn create_schedule(
350        &self,
351        document: &mut Value,
352        payload: &Map<String, Value>,
353        owner: Option<&str>,
354        host: &dyn TaskHost,
355    ) -> Result<Value, String> {
356        kernel::ops::fields("schedules.create", payload)?;
357        let definition = definition_key(payload.get("definition"), "definition")?;
358        let input = payload.get("input").cloned().ok_or("input must be JSON")?;
359        let every_ms = integer(payload.get("every_ms"), "every_ms", 1, MAX_EVERY_MS)?;
360        let now = self.now(host);
361        let start_at = optional_integer(payload.get("start_at"), "start_at", 0)?.unwrap_or(now);
362        let max_runs = optional_integer(payload.get("max_runs"), "max_runs", 1)?;
363        let model = match payload.get("model") {
364            None | Some(Value::Null) => Value::Null,
365            Some(model) => model_scope::model_ref(model)?.into(),
366        };
367        let goal_id = payload.get("goal_id").cloned().unwrap_or(Value::Null);
368        self.update(document, |draft| {
369            if !goal_id.is_null() && lookup(draft, "goals", Some(&goal_id), "Unknown goal")?["state"] == "achieved" {
370                return Err("Cannot attach a schedule to an achieved goal".into());
371            }
372            let schedule = json!({
373                "id": format!("schedule-{}", jsrt::random::uuid_v4()),
374                "definition": definition, "input": input, "every_ms": every_ms, "start_at": start_at,
375                "max_runs": max_runs, "goal_id": goal_id, "model": model, "owner": owner, "state": "active",
376                "created_at": now, "runs": 0, "last_slot": null, "pending_slot": null, "last_task_id": null,
377                "last_fired_at": null, "last_skipped": 0, "skipped_total": 0, "last_failure": null, "failures": 0,
378            });
379            if let Some(items) = draft["schedules"].as_array_mut() {
380                items.push(schedule.clone());
381            }
382            Ok(schedule)
383        })
384    }
385
386    fn set_schedule_state(
387        &self,
388        document: &mut Value,
389        kind: &str,
390        payload: &Map<String, Value>,
391        state: &str,
392    ) -> Result<Value, String> {
393        kernel::ops::fields(kind, payload)?;
394        self.update(document, |draft| {
395            let id = lookup(draft, "schedules", payload.get("id"), "Unknown schedule")?["id"].clone();
396            let schedule = find(draft, "schedules", id.as_str().unwrap_or("")).expect("found above");
397            if schedule["state"] == "completed" {
398                return Err("Schedule is completed".into());
399            }
400            schedule["state"] = state.into();
401            Ok(schedule.clone())
402        })
403    }
404
405    fn create_goal(
406        &self,
407        document: &mut Value,
408        payload: &Map<String, Value>,
409        owner: Option<&str>,
410        host: &dyn TaskHost,
411    ) -> Result<Value, String> {
412        kernel::ops::fields("goals.create", payload)?;
413        let title = jsrt::payload::nonempty_string(payload.get("title"), "title")?;
414        let items = payload
415            .get("required_checks")
416            .and_then(Value::as_array)
417            .ok_or("required_checks must be an array")?;
418        let checks = items
419            .iter()
420            .map(|item| definition_key(Some(item), "required_checks entry"))
421            .collect::<Result<Vec<_>, _>>()?;
422        if checks.iter().collect::<HashSet<_>>().len() != checks.len() {
423            return Err("required_checks must not repeat".into());
424        }
425        let max_tasks = optional_integer(payload.get("max_tasks"), "max_tasks", 0)?;
426        let now = self.now(host);
427        self.update(document, |draft| {
428            let goal = json!({
429                "id": format!("goal-{}", jsrt::random::uuid_v4()), "title": title, "required_checks": checks, "max_tasks": max_tasks,
430                "owner": owner, "state": "active", "created_at": now, "task_ids": [], "verify_rounds": 0,
431                "verification": null,
432            });
433            if let Some(items) = draft["goals"].as_array_mut() {
434                items.push(goal.clone());
435            }
436            Ok(goal)
437        })
438    }
439
440    fn set_goal_state(
441        &self,
442        document: &mut Value,
443        kind: &str,
444        payload: &Map<String, Value>,
445        state: &str,
446    ) -> Result<Value, String> {
447        kernel::ops::fields(kind, payload)?;
448        self.update(document, |draft| {
449            let id = lookup(draft, "goals", payload.get("id"), "Unknown goal")?["id"].clone();
450            let goal = find(draft, "goals", id.as_str().unwrap_or("")).expect("found above");
451            if goal["state"] == "achieved" {
452                return Err("Goal is already achieved".into());
453            }
454            goal["state"] = state.into();
455            Ok(goal.clone())
456        })
457    }
458
459    fn attach(&self, document: &mut Value, payload: &Map<String, Value>, host: &dyn TaskHost) -> Result<Value, String> {
460        kernel::ops::fields("goals.attach", payload)?;
461        let task_id = jsrt::payload::nonempty_string(payload.get("task_id"), "task_id")?;
462        let goal_id = lookup(document, "goals", payload.get("id"), "Unknown goal")?["id"].clone();
463        if !host.tasks()?.iter().any(|task| task["id"] == task_id.as_str()) {
464            return Err("Unknown Ultron task".into());
465        }
466        self.update(document, |draft| {
467            let goal = find(draft, "goals", goal_id.as_str().unwrap_or("")).ok_or("Unknown goal")?;
468            let task = Value::from(task_id.clone());
469            if !goal["task_ids"].as_array().is_some_and(|ids| ids.contains(&task)) {
470                if !has_capacity(goal, 1) {
471                    return Err(format!("Goal task limit reached ({})", goal["max_tasks"]));
472                }
473                push_task(goal, &task);
474            }
475            Ok(goal.clone())
476        })
477    }
478
479    /// Fires due schedules (the TS background timer's work); returns `{fired, failed}`.
480    pub fn tick(&self, host: &dyn TaskHost) -> Result<Value, String> {
481        let mut guard = jsrt::sync::lock(&self.document);
482        let document = self.loaded(&mut guard)?;
483        self.tick_locked(document, host)
484    }
485
486    fn tick_locked(&self, document: &mut Value, host: &dyn TaskHost) -> Result<Value, String> {
487        let mut fired = Vec::new();
488        let mut failed = Vec::new();
489        let now = self.now(host);
490        let ids: Vec<String> = document["schedules"]
491            .as_array()
492            .into_iter()
493            .flatten()
494            .filter_map(|item| item["id"].as_str().map(str::to_string))
495            .collect();
496        for id in ids {
497            let Some(current) = document["schedules"]
498                .as_array()
499                .and_then(|items| items.iter().find(|item| item["id"] == id.as_str()))
500                .cloned()
501            else {
502                continue;
503            };
504            if current["state"] != "active" {
505                continue;
506            }
507            let pending = current["pending_slot"].as_i64();
508            let slot = pending.unwrap_or_else(|| {
509                schedule_slot(
510                    current["start_at"].as_i64().unwrap_or(0),
511                    current["every_ms"].as_i64().unwrap_or(1),
512                    now,
513                )
514            });
515            if slot < 0 {
516                continue;
517            }
518            let resuming = pending.is_some();
519            let last = current["last_slot"].as_i64();
520            if !resuming && last.is_some_and(|last| slot <= last) {
521                continue;
522            }
523            let goal = current["goal_id"].as_str().and_then(|goal_id| {
524                document["goals"]
525                    .as_array()
526                    .and_then(|goals| goals.iter().find(|goal| goal["id"] == goal_id))
527                    .cloned()
528            });
529            // A missing goal cannot be verified or paused, so its schedules stay idle.
530            if !current["goal_id"].is_null() && goal.as_ref().is_none_or(|goal| goal["state"] != "active") {
531                continue;
532            }
533            let skipped = if resuming {
534                current["last_skipped"].as_i64().unwrap_or(0)
535            } else {
536                slot - last.unwrap_or(-1) - 1
537            };
538            let claimed = self.update(document, |draft| {
539                let schedule = find(draft, "schedules", &id).ok_or("Unknown schedule")?;
540                schedule["last_slot"] = slot.into();
541                schedule["pending_slot"] = slot.into();
542                schedule["last_skipped"] = skipped.into();
543                if !resuming {
544                    schedule["skipped_total"] = (schedule["skipped_total"].as_i64().unwrap_or(0) + skipped).into();
545                }
546                Ok(schedule.clone())
547            })?;
548            let mut task = None;
549            let mut failure = None;
550            if let Some(goal) = goal.as_ref().filter(|goal| !has_capacity(goal, 1)) {
551                failure = Some(format!("Goal task limit reached ({})", goal["max_tasks"]));
552            } else {
553                let mut request = json!({
554                    "definition": claimed["definition"], "input": claimed["input"], "key": schedule_key(&claimed, slot),
555                });
556                if !claimed["model"].is_null() {
557                    request["model"] = claimed["model"].clone();
558                }
559                match host.spawn(&request, claimed["owner"].as_str()) {
560                    Ok(spawned) => task = Some(spawned),
561                    Err(error) => failure = Some(error),
562                }
563            }
564            // The slot stays claimed on failure: the next attempt is the next slot, never a retry loop.
565            self.update(document, |draft| {
566                let schedule = find(draft, "schedules", &id).ok_or("Unknown schedule")?;
567                schedule["pending_slot"] = Value::Null;
568                let mut linked = None;
569                match &task {
570                    Some(task) => {
571                        let runs = schedule["runs"].as_i64().unwrap_or(0) + 1;
572                        schedule["runs"] = runs.into();
573                        schedule["last_task_id"] = task["id"].clone();
574                        schedule["last_fired_at"] = now.into();
575                        if schedule["max_runs"].as_i64().is_some_and(|max| runs >= max) {
576                            schedule["state"] = "completed".into();
577                        }
578                        linked = schedule["goal_id"].as_str().map(str::to_string);
579                    }
580                    None => {
581                        schedule["failures"] = (schedule["failures"].as_i64().unwrap_or(0) + 1).into();
582                        schedule["last_failure"] = json!({"slot": slot,
583                            "error": failure.clone().unwrap_or_else(|| "unknown failure".into()), "at": now});
584                    }
585                }
586                if let (Some(goal_id), Some(task)) = (linked, &task)
587                    && let Some(goal) = find(draft, "goals", &goal_id)
588                {
589                    push_task(goal, &task["id"]);
590                }
591                Ok(())
592            })?;
593            match task {
594                Some(task) => {
595                    fired.push(json!({"schedule_id": id, "slot": slot, "task_id": task["id"], "skipped": skipped}))
596                }
597                None => failed.push(json!({"schedule_id": id, "slot": slot,
598                    "error": failure.unwrap_or_else(|| "unknown failure".into())})),
599            }
600        }
601        Ok(json!({"fired": fired, "failed": failed}))
602    }
603
604    /// Runs every required check and accepts the goal only on unanimous, explicit passes. Admission is serialized
605    /// with other mutations; waiting for results is not.
606    fn verify(&self, payload: &Map<String, Value>, caller: Option<&str>, host: &dyn TaskHost) -> Result<Value, String> {
607        kernel::ops::fields("goals.verify", payload)?;
608        let (goal_id, round, spawned) = {
609            let mut guard = jsrt::sync::lock(&self.document);
610            let document = self.loaded(&mut guard)?;
611            let goal = lookup(document, "goals", payload.get("id"), "Unknown goal")?;
612            if goal["state"] == "achieved" {
613                return Err("Goal is already achieved".into());
614            }
615            if goal["state"] == "paused" {
616                return Err("Goal is paused".into());
617            }
618            let checks: Vec<String> = goal["required_checks"]
619                .as_array()
620                .into_iter()
621                .flatten()
622                .filter_map(|check| check.as_str().map(str::to_string))
623                .collect();
624            if !has_capacity(&goal, checks.len() as i64) {
625                return Err(format!(
626                    "Goal task limit reached: verification needs {} tasks, {} remain",
627                    checks.len(),
628                    capacity(&goal).unwrap_or(0).max(0)
629                ));
630            }
631            let goal_id = goal["id"].as_str().unwrap_or("").to_string();
632            let round = self.update(document, |draft| {
633                let target = find(draft, "goals", &goal_id).ok_or("Unknown goal")?;
634                let round = target["verify_rounds"].as_i64().unwrap_or(0) + 1;
635                target["verify_rounds"] = round.into();
636                Ok(round)
637            })?;
638            let input = payload
639                .get("input")
640                .cloned()
641                .unwrap_or_else(|| json!({"goal_id": goal_id, "title": goal["title"]}));
642            let spawned: Vec<(String, Result<Value, String>)> = checks
643                .iter()
644                .enumerate()
645                .map(|(index, definition)| {
646                    let request = json!({"definition": definition, "input": input,
647                        "key": format!("goal:{goal_id}:verify:{round}:{index}")});
648                    (definition.clone(), host.spawn(&request, caller))
649                })
650                .collect();
651            self.update(document, |draft| {
652                let target = find(draft, "goals", &goal_id).ok_or("Unknown goal")?;
653                for (_, task) in &spawned {
654                    if let Ok(task) = task {
655                        push_task(target, &task["id"]);
656                    }
657                }
658                Ok(())
659            })?;
660            (goal_id, round, spawned)
661        };
662        let checks: Vec<Value> = spawned
663            .iter()
664            .map(|(definition, task)| match task {
665                Err(error) => missing(definition, None, error),
666                Ok(task) => {
667                    let id = task["id"].as_str().unwrap_or("");
668                    match host.result(id) {
669                        Ok(result) => check_outcome(definition, Some(id), &result),
670                        Err(error) => missing(definition, Some(id), &error),
671                    }
672                }
673            })
674            .collect();
675        let mut guard = jsrt::sync::lock(&self.document);
676        let document = self.loaded(&mut guard)?;
677        let now = self.now(host);
678        self.update(document, |draft| {
679            let goal = find(draft, "goals", &goal_id).ok_or("Unknown goal")?;
680            let mut reasons: Vec<Value> = checks.iter().filter_map(|check| check.get("reason").cloned()).collect();
681            let status = if checks.is_empty() {
682                reasons.push("Goal has no required checks".into());
683                "unverified"
684            } else if checks.iter().all(|check| check["passed"] == true) {
685                "achieved"
686            } else {
687                "unachieved"
688            };
689            // A newer round supersedes this one; its record is the one that counts.
690            if goal["verification"]["round"]
691                .as_i64()
692                .is_none_or(|recorded| recorded < round)
693            {
694                goal["verification"] =
695                    json!({"round": round, "status": status, "at": now, "checks": checks, "reasons": reasons});
696                let state = verified_state(goal["state"].as_str().unwrap_or_default(), status);
697                goal["state"] = state.into();
698            }
699            Ok(goal.clone())
700        })
701    }
702}
703
704/// A goal's state once a verification round with `status` is recorded (the latest round decides, the check results
705/// arrive with the lock released): an active goal it passes is achieved, an achieved goal it does not pass (an
706/// older round passed first) is active again, and a goal paused meanwhile stays paused.
707fn verified_state<'a>(state: &'a str, status: &str) -> &'a str {
708    match (state, status) {
709        ("active", "achieved") => "achieved",
710        ("achieved", status) if status != "achieved" => "active",
711        (state, _) => state,
712    }
713}
714
715impl crate::ApiModule for SchedulesModule {
716    fn prefixes(&self) -> &[&str] {
717        &["schedules.", "goals."]
718    }
719
720    fn start(&self, host: &dyn TaskHost) -> Result<(), String> {
721        SchedulesModule::start(self, host)
722    }
723
724    fn handle(&self, request: &crate::Request<'_>) -> Result<Value, String> {
725        SchedulesModule::handle(self, request.kind, request.payload, request.caller_task, request.host)
726    }
727}