Skip to main content

kernel_apis/
cell_hints.rs

1//! Situational hints (`ultron/rlm/hints.ts` `CellHints`, `ultron/rlm/loop-detector.ts`): the host watches how a
2//! lane uses the runtime and ends a cell's result with at most one short tagged line, `[hint:<tag>] ...`, when
3//! something worth knowing happened in that cell: a command detached into a job, a long wait on a job whose
4//! completion would have arrived as an event, polling, truncated output, a large file read into a string, the same
5//! exception three cells in a row, the same cell run again and again, or a lane stuck in a loop of failing or
6//! alternating cells. Mutes and per-tag counts are [`HintsModule`]'s (`hints.mute`). Hints are off unless
7//! `ULTRON_HINTS=on`.
8
9use crate::hints::HintsModule;
10use crate::util::{Clock, system_clock};
11use fancy_regex::Regex;
12use serde_json::{Map, Value};
13use std::collections::{HashMap, HashSet};
14use std::sync::{Arc, LazyLock, Mutex};
15
16pub const HINTS_ENV: &str = "ULTRON_HINTS";
17pub const HINTS_MAX_ENV: &str = "ULTRON_HINTS_MAX";
18
19pub const DEFAULT_HINT_MAX_PER_TAG: u64 = 3;
20pub const DEFAULT_BLOCKED_SECONDS: f64 = 60.0;
21pub const DEFAULT_READ_HANDLE_BYTES: u64 = 256 * 1024;
22/// Same exception type this many cells in a row.
23const FAILURE_STREAK: usize = 3;
24/// The same successful cell this many times in a row (and again at twice as many).
25const CELL_REPEATS: usize = 3;
26/// The hints that fire without `ULTRON_HINTS` ([`CellHintsOptions::guards`]): a lane going in circles.
27const GUARD_TAGS: &[&str] = &["stuck-loop", "repeated-cell"];
28/// Same status call this many times in one cell.
29const POLL_REPEATS: usize = 3;
30
31/// Waits that hold the cell until detached work ends. Waiting for subagents earns no hint (the guide asks for it).
32const BLOCKING_WAITS: &[&str] = &["shell.result", "background.result"];
33
34/// Status checks of running work; repeating one is polling.
35const STATUS_CALLS: &[&str] = &[
36    "shell.get",
37    "shell.result",
38    "shell.list",
39    "background.inspect",
40    "background.list",
41    "agents.inspect",
42    "agents.tasks",
43    "agents.status",
44    "rlm.list_subagents",
45];
46
47fn regex(pattern: &str) -> Regex {
48    Regex::new(pattern).expect("valid hint pattern")
49}
50
51static TRUNCATION_MARKER: LazyLock<Regex> =
52    LazyLock::new(|| regex(r"\[\.\.\. \d+ bytes truncated(?:; [^\n]*?)? \.\.\.\]"));
53static FILE_READ: LazyLock<Regex> =
54    LazyLock::new(|| regex(r"\.read\(\s*\)|\.read_text\(|\.read_bytes\(|\.readlines\(|\bopen\("));
55static BIG_STRING: LazyLock<Regex> = LazyLock::new(|| regex(r"(?m)^<str: ([\d,]+) chars"));
56static SLEEP: LazyLock<Regex> = LazyLock::new(|| regex(r"\b(?:time|asyncio)\.sleep\("));
57static LOOP: LazyLock<Regex> = LazyLock::new(|| regex(r"(?m)^\s*(?:while|for|async\s+for)\b"));
58static STATUS_CHECK: LazyLock<Regex> = LazyLock::new(|| {
59    regex(
60        r"\.running\b|\.status\b|\.done\(|\.result\(|\.inspect\(|rlm\.jobs\(|rlm\.job\(|list_subagents\(|agents\.tasks\(|agents\.status\(|background\.list\(",
61    )
62});
63static QUERY_COMMANDS: LazyLock<Regex> = LazyLock::new(|| regex(r"^\s*(?:grep|egrep|fgrep|rg|diff|cmp|test|\[)\b"));
64
65fn matches(re: &Regex, text: &str) -> bool {
66    re.is_match(text).unwrap_or(false)
67}
68
69/// Hints are off unless ULTRON_HINTS is on (or 1, true, yes).
70pub fn hints_enabled(value: Option<&str>) -> bool {
71    value.is_some_and(|v| ["on", "1", "true", "yes"].contains(&v.trim().to_lowercase().as_str()))
72}
73
74/// `hintMaxPerTag`: ULTRON_HINTS_MAX, a non-negative integer, else 3.
75pub fn hint_max_per_tag(value: Option<&str>) -> u64 {
76    value
77        .map(str::trim)
78        .filter(|v| !v.is_empty())
79        .and_then(|v| v.parse::<u64>().ok())
80        .unwrap_or(DEFAULT_HINT_MAX_PER_TAG)
81}
82
83/// `readHandleBytes`: ULTRON_READ_HANDLE_BYTES, a positive integer, else 256 KiB.
84pub fn read_handle_bytes(value: Option<&str>) -> u64 {
85    value
86        .and_then(|v| v.trim().parse::<u64>().ok())
87        .filter(|v| *v > 0)
88        .unwrap_or(DEFAULT_READ_HANDLE_BYTES)
89}
90
91/// Whether a finished `bash` command failed: a timeout, or a non-zero exit (1 is an answer for a query command).
92pub fn bash_failed(command: &str, exit_code: &Value, timed_out: bool) -> bool {
93    if timed_out {
94        return true;
95    }
96    match exit_code.as_f64() {
97        None => false,
98        Some(0.0) => false,
99        Some(code) => !(code == 1.0 && matches(&QUERY_COMMANDS, command)),
100    }
101}
102
103/// Thresholds of the stuck-loop detector (`LoopLimits`).
104#[derive(Clone, Copy, Debug, PartialEq)]
105pub struct LoopLimits {
106    /// Failed cells in a row.
107    pub consecutive: usize,
108    /// Cells in an A, B, A, B run.
109    pub alternation: usize,
110    /// Failed cells in one run.
111    pub total: usize,
112}
113
114impl Default for LoopLimits {
115    fn default() -> Self {
116        Self {
117            consecutive: 3,
118            alternation: 4,
119            total: 8,
120        }
121    }
122}
123
124#[derive(Clone, Debug, PartialEq)]
125pub struct LoopTrip {
126    pub level: u8,
127    /// `consecutive`, `alternation` or `total`.
128    pub reason: &'static str,
129    pub count: usize,
130    pub ename: Option<String>,
131}
132
133#[derive(Default)]
134struct LaneLoop {
135    history: Vec<(String, bool)>,
136    consecutive: usize,
137    streak_ename: Option<String>,
138    total: usize,
139    level: u8,
140    total_level: u8,
141}
142
143/// Code with whitespace and digits removed, so cells differing only in those compare equal.
144pub fn normalize_cell(code: &str) -> String {
145    code.chars()
146        .filter(|c| !c.is_whitespace() && !c.is_ascii_digit())
147        .collect()
148}
149
150/// Length of the A, B, A, B run (A differs from B) that ends the history; 0 when there is none.
151pub fn alternation_length(hashes: &[&str]) -> usize {
152    let n = hashes.len();
153    if n < 2 || hashes[n - 1] == hashes[n - 2] {
154        return 0;
155    }
156    let mut length = 2;
157    let mut i = n as isize - 3;
158    while i >= 0 && hashes[i as usize] == hashes[i as usize + 2] {
159        length += 1;
160        i -= 1;
161    }
162    length
163}
164
165/// `ToolLoopDetector`.
166#[derive(Default)]
167pub struct LoopDetector {
168    limits: LoopLimits,
169    lanes: HashMap<String, LaneLoop>,
170}
171
172impl LoopDetector {
173    pub fn new(limits: LoopLimits) -> Self {
174        Self {
175            limits,
176            lanes: HashMap::new(),
177        }
178    }
179
180    pub fn level(&self, lane: &str) -> u8 {
181        self.lanes.get(lane).map_or(0, |s| s.level)
182    }
183
184    /// A cell on `lane` ended; a trip when this cell makes the lane look stuck.
185    pub fn cell_ended(&mut self, lane: &str, code: &str, failed: bool, ename: Option<&str>) -> Option<LoopTrip> {
186        let limits = self.limits;
187        let state = self.lanes.entry(lane.to_string()).or_default();
188        let hash = jsrt::hash::sha256_hex(normalize_cell(code).as_bytes())[..16].to_string();
189        state.history.push((hash.clone(), failed));
190        if state.history.len() > limits.alternation * 2 + 2 {
191            state.history.remove(0);
192        }
193        let ename = ename.unwrap_or("").to_string();
194        if failed {
195            state.consecutive += 1;
196            state.total += 1;
197            state.streak_ename = Some(
198                if state.consecutive == 1 || state.streak_ename.as_deref() == Some(&ename) {
199                    ename
200                } else {
201                    String::new()
202                },
203            );
204        } else {
205            state.consecutive = 0;
206            state.streak_ename = None;
207        }
208        let n = state.history.len();
209        let repeat = n >= 3 && state.history[n - 3].0 == hash;
210        // Progress: a success that is not part of a back-and-forth ends the phase.
211        if !failed && !repeat {
212            state.level = 0;
213        }
214        let hashes: Vec<&str> = state.history.iter().map(|(h, _)| h.as_str()).collect();
215        let run = alternation_length(&hashes);
216        let run_failures = state.history[n - run..].iter().filter(|(_, f)| *f).count();
217        let alternating = run >= limits.alternation && run_failures * 2 >= run;
218        let trip = |state: &mut LaneLoop, level: u8, reason: &'static str, count: usize| {
219            state.level = level;
220            if reason == "total" {
221                state.total_level = level;
222            }
223            let ename = (reason == "consecutive")
224                .then(|| state.streak_ename.clone().filter(|e| !e.is_empty()))
225                .flatten();
226            Some(LoopTrip {
227                level,
228                reason,
229                count,
230                ename,
231            })
232        };
233        if state.level == 0 {
234            if limits.consecutive > 0 && state.consecutive >= limits.consecutive {
235                let count = state.consecutive;
236                return trip(state, 1, "consecutive", count);
237            }
238            if limits.alternation > 0 && alternating {
239                return trip(state, 1, "alternation", run);
240            }
241        } else if state.level == 1 {
242            if limits.consecutive > 0 && state.consecutive >= limits.consecutive * 2 {
243                let count = state.consecutive;
244                return trip(state, 2, "consecutive", count);
245            }
246            if limits.alternation > 0 && alternating && run >= limits.alternation * 2 {
247                return trip(state, 2, "alternation", run);
248            }
249        }
250        // The run total spans phases: it trips once, and escalates once, per run.
251        if limits.total > 0 && failed {
252            let total = state.total;
253            if state.total_level == 0 && state.level == 0 && total >= limits.total {
254                return trip(state, 1, "total", total);
255            }
256            if state.total_level == 1 && state.level < 2 && total >= limits.total * 2 {
257                return trip(state, 2, "total", total);
258            }
259        }
260        None
261    }
262
263    pub fn reset(&mut self, lane: &str) {
264        self.lanes.remove(lane);
265    }
266}
267
268/// The `stuck-loop` hint text for a trip.
269pub fn loop_hint_text(trip: &LoopTrip) -> String {
270    let what = match trip.reason {
271        "consecutive" => format!(
272            "The last {} cells failed{}",
273            trip.count,
274            trip.ename
275                .as_ref()
276                .map_or(String::new(), |e| format!(" ({e} each time)"))
277        ),
278        "alternation" => format!(
279            "The last {} cells alternate between two near-identical versions",
280            trip.count
281        ),
282        _ => format!("{} cells have failed in this run", trip.count),
283    };
284    if trip.level == 1 {
285        format!(
286            "{what}; you look stuck. Before the next cell, state three different hypotheses for why it fails, then run one cell that tests the most likely one instead of retrying a variant."
287        )
288    } else {
289        format!(
290            "{what}, even after the last hint. Stop repeating this approach: name three hypotheses you have not tested yet and test the most likely; if none holds, say what blocks you."
291        )
292    }
293}
294
295#[derive(Clone)]
296pub struct CellHintsOptions {
297    pub enabled: bool,
298    /// Loop guards, whether or not hints are `enabled`: the `stuck-loop` and `repeated-cell` hints fire, and a cell
299    /// that already ran twice [`CELL_REPEATS`] times in a row is refused ([`CellHints::refusal`]).
300    pub guards: bool,
301    pub max_per_tag: u64,
302    /// Completions are announced (ULTRON_ASYNC_EVENTS): the wait and polling hints point at events.
303    pub async_events: bool,
304    /// `rootDelivery: "next-call"`: nothing wakes the root lane between turns. Without
305    /// [`CellHints::set_lane_woken`] only the root lane (`main`) counts as woken, and only without this.
306    pub next_call_root: bool,
307    pub blocked_seconds: f64,
308    pub read_handle_bytes: u64,
309    pub loop_limits: LoopLimits,
310    pub now: Clock,
311}
312
313impl Default for CellHintsOptions {
314    fn default() -> Self {
315        Self {
316            enabled: true,
317            guards: true,
318            max_per_tag: DEFAULT_HINT_MAX_PER_TAG,
319            async_events: true,
320            next_call_root: false,
321            blocked_seconds: DEFAULT_BLOCKED_SECONDS,
322            read_handle_bytes: DEFAULT_READ_HANDLE_BYTES,
323            loop_limits: LoopLimits::default(),
324            now: system_clock(),
325        }
326    }
327}
328
329/// How a cell ended, as the rlm tool reports it.
330pub struct CellOutcome<'a> {
331    /// The result text (after truncation).
332    pub text: &'a str,
333    /// Exception type when the cell failed.
334    pub ename: Option<&'a str>,
335}
336
337#[derive(Default)]
338struct Cell {
339    code: String,
340    /// (job id, the yield it waited) of each command that came back running.
341    detached: Vec<(String, Option<f64>)>,
342    waits: Vec<(i64, i64)>,
343    wait_types: Vec<String>,
344    /// Status call key → count, in first-call order.
345    status_calls: Vec<(String, usize)>,
346    bash_failed: bool,
347}
348
349#[derive(Default)]
350struct LaneState {
351    cell: Option<Cell>,
352    previous_status: HashSet<String>,
353    failure: Option<(String, usize)>,
354    repeat: Option<(String, usize)>,
355}
356
357/// Whether the host wakes a lane when its work ends (its event wording says "end your turn").
358pub type LaneWoken = Arc<dyn Fn(&str) -> bool + Send + Sync>;
359
360pub struct CellHints {
361    module: Arc<HintsModule>,
362    options: CellHintsOptions,
363    lanes: Mutex<HashMap<String, LaneState>>,
364    loops: Mutex<LoopDetector>,
365    woken: Mutex<Option<LaneWoken>>,
366}
367
368impl CellHints {
369    pub fn new(module: Arc<HintsModule>, options: CellHintsOptions) -> Self {
370        let loops = LoopDetector::new(options.loop_limits);
371        Self {
372            module,
373            options,
374            lanes: Mutex::new(HashMap::new()),
375            loops: Mutex::new(loops),
376            woken: Mutex::new(None),
377        }
378    }
379
380    /// Which lanes the host wakes when their work ends: the others' hints say to await what the answer needs
381    /// (nothing wakes them after they reply) instead of "end your turn".
382    pub fn set_lane_woken(&self, woken: LaneWoken) {
383        *jsrt::sync::lock(&self.woken) = Some(woken);
384    }
385
386    /// Whether `lane` is woken by the ends of its work.
387    fn woken(&self, lane: &str) -> bool {
388        let woken = jsrt::sync::lock(&self.woken).clone();
389        match woken {
390            Some(woken) => woken(lane),
391            None => lane == kernel::ROOT_LANE && !self.options.next_call_root,
392        }
393    }
394
395    pub fn enabled(&self) -> bool {
396        self.options.enabled
397    }
398
399    /// Whether cells are observed at all: every hint is on, or the loop guards are.
400    pub fn active(&self) -> bool {
401        self.options.enabled || self.options.guards
402    }
403
404    /// Why `code` is not run on `lane`: the same cell just succeeded twice [`CELL_REPEATS`] times in a row there
405    /// (the `repeated-cell` hint said so twice). Any other cell starts the count over.
406    pub fn refusal(&self, lane: &str, code: &str) -> Option<String> {
407        if !self.options.guards {
408            return None;
409        }
410        let lanes = jsrt::sync::lock(&self.lanes);
411        let (previous, count) = lanes.get(lane)?.repeat.as_ref()?;
412        (*count >= CELL_REPEATS * 2 && previous == code.trim()).then(|| {
413            format!(
414                "Not run: this exact cell already ran {count} times in a row and succeeded each time. Use the output you have, run something different, or end your turn and wait for the event."
415            )
416        })
417    }
418
419    fn now(&self) -> i64 {
420        (self.options.now)()
421    }
422
423    /// `lane`'s run ended: failure streaks and repeats start over with its next run.
424    pub fn run_ended(&self, lane: &str) {
425        jsrt::sync::lock(&self.loops).reset(lane);
426        if let Some(state) = jsrt::sync::lock(&self.lanes).get_mut(lane) {
427            state.failure = None;
428            state.repeat = None;
429        }
430    }
431
432    /// A cell starts on `lane`.
433    pub fn begin_cell(&self, lane: &str, code: &str) {
434        if !self.active() {
435            return;
436        }
437        jsrt::sync::lock(&self.lanes).entry(lane.to_string()).or_default().cell = Some(Cell {
438            code: code.to_string(),
439            ..Cell::default()
440        });
441    }
442
443    /// A host request from `lane`'s kernel finished (`result` is `None` when it failed).
444    pub fn observe(
445        &self,
446        lane: &str,
447        kind: &str,
448        payload: &Map<String, Value>,
449        result: Option<&Value>,
450        started_at: i64,
451    ) {
452        let now = self.now();
453        let mut lanes = jsrt::sync::lock(&self.lanes);
454        let Some(cell) = lanes.get_mut(lane).and_then(|s| s.cell.as_mut()) else {
455            return;
456        };
457        let reply = result.and_then(Value::as_object);
458        let running = reply.is_some_and(|r| r.get("running") == Some(&Value::Bool(true)));
459        if kind == "bash"
460            && let Some(reply) = reply
461            && !running
462        {
463            cell.bash_failed = bash_failed(
464                payload.get("command").and_then(Value::as_str).unwrap_or(""),
465                reply.get("exit_code").unwrap_or(&Value::Null),
466                reply.get("timed_out") == Some(&Value::Bool(true)),
467            );
468        }
469        if kind == "bash" && running {
470            let id = reply
471                .and_then(|r| r.get("job"))
472                .and_then(|j| j.get("id"))
473                .and_then(Value::as_str)
474                .unwrap_or("a job");
475            let waited = payload.get("yield_after").and_then(Value::as_f64);
476            cell.detached.push((id.to_string(), waited));
477        }
478        if BLOCKING_WAITS.contains(&kind) {
479            cell.waits.push((started_at, now));
480            if !cell.wait_types.iter().any(|t| t == kind) {
481                cell.wait_types.push(kind.to_string());
482            }
483        }
484        if STATUS_CALLS.contains(&kind) {
485            // A status call that found the work finished is a result fetch, not a poll.
486            if (kind == "shell.get" || kind == "shell.result") && !running {
487                return;
488            }
489            let key = match payload.get("id").and_then(Value::as_str) {
490                Some(id) => format!("{kind}:{id}"),
491                None => kind.to_string(),
492            };
493            match cell.status_calls.iter_mut().find(|(k, _)| *k == key) {
494                Some(entry) => entry.1 += 1,
495                None => cell.status_calls.push((key, 1)),
496            }
497        }
498    }
499
500    /// The cell on `lane` ended: the hint line to append to its result, if any.
501    pub fn end_cell(&self, lane: &str, outcome: &CellOutcome<'_>) -> Option<String> {
502        if !self.active() {
503            return None;
504        }
505        // A lane nothing wakes gets the hints that say to await what the answer needs.
506        let next_call = !self.woken(lane);
507        let (mut candidates, code, failed) = {
508            let mut lanes = jsrt::sync::lock(&self.lanes);
509            let state = lanes.entry(lane.to_string()).or_default();
510            let cell = state.cell.take()?;
511            let candidates = self.candidates(state, &cell, outcome, next_call);
512            state.previous_status = cell.status_calls.iter().map(|(k, _)| k.clone()).collect();
513            (candidates, cell.code, outcome.ename.is_some() || cell.bash_failed)
514        };
515        let (trip, level) = {
516            let mut loops = jsrt::sync::lock(&self.loops);
517            let trip = loops.cell_ended(lane, &code, failed, outcome.ename);
518            (trip, loops.level(lane))
519        };
520        if let Some(trip) = trip {
521            candidates.insert(0, ("stuck-loop", loop_hint_text(&trip)));
522        }
523        if !self.options.enabled {
524            candidates.retain(|(tag, _)| GUARD_TAGS.contains(tag));
525        }
526        if candidates.is_empty() {
527            return None;
528        }
529        self.module.fire(lane, &candidates, self.options.max_per_tag, level > 0)
530    }
531
532    /// Every hint this cell earned, most specific first.
533    fn candidates(
534        &self,
535        state: &mut LaneState,
536        cell: &Cell,
537        outcome: &CellOutcome<'_>,
538        next_call: bool,
539    ) -> Vec<(&'static str, String)> {
540        let events = self.options.async_events;
541        let mut found: Vec<(&'static str, String)> = Vec::new();
542        // Failure streaks count every cell, whether or not a hint fires for it.
543        state.failure = match (outcome.ename, state.failure.take()) {
544            (Some(ename), Some((previous, count))) if previous == ename => Some((previous, count + 1)),
545            (Some(ename), _) => Some((ename.to_string(), 1)),
546            (None, _) => None,
547        };
548        // Exact repeats only: cells that differ in numbers (paging through a list) are different work.
549        let code = cell.code.trim().to_string();
550        state.repeat = if outcome.ename.is_none() && !cell.bash_failed {
551            match state.repeat.take() {
552                Some((previous, count)) if previous == code => Some((code, count + 1)),
553                _ => Some((code, 1)),
554            }
555        } else {
556            None
557        };
558        let repeat = state
559            .repeat
560            .as_ref()
561            .filter(|(_, count)| *count == CELL_REPEATS || *count == CELL_REPEATS * 2)
562            .map(|(_, count)| {
563                (
564                    "repeated-cell",
565                    format!(
566                        "This exact cell has now run {count} times in a row and succeeded each time; rerunning it will not tell you more. Use the output you have, keep values in variables instead of recomputing them, and wait for running work once (`await job.result()`, `await rlm.collect(...)`) instead of checking again."
567                    ),
568                )
569            });
570
571        if let Some((id, waited)) = cell.detached.first() {
572            let after = waited.map_or(String::new(), |w| format!(" after {} s", jsrt::number::to_string(w)));
573            found.push((
574                "job-detached",
575                if events && next_call {
576                    format!(
577                        "The command was still running{after}, so it continues as job {id}. Do other work meanwhile: its completion is reported as a <runtime_event> at the top of a later rlm result, and `await <result>.job.result()` waits for it when your answer needs it (nothing wakes you after you reply). (`yield_after=None` blocks until a command ends; `yield_after=0` detaches at once.)"
578                    )
579                } else if events {
580                    format!(
581                        "The command was still running{after}, so it continues as job {id}. Do not wait for it: do other work or end your turn; its completion arrives as a <runtime_event> that starts your next turn. (`yield_after=None` blocks until a command ends; `yield_after=0` detaches at once.)"
582                    )
583                } else {
584                    format!(
585                        "The command was still running{after}, so it continues as job {id}; `await <result>.job.result()` waits for it. (`yield_after=None` blocks until a command ends.)"
586                    )
587                },
588            ));
589        }
590        let waited_ms = union_ms(&cell.waits);
591        // Without wake-ups, waiting for a result the answer needs is the right move, not a mistake.
592        if events && !next_call && waited_ms as f64 >= self.options.blocked_seconds * 1000.0 {
593            let calls: Vec<&str> = cell
594                .wait_types
595                .iter()
596                .map(|t| {
597                    if t == "shell.result" {
598                        "job.result()"
599                    } else {
600                        "background.result()"
601                    }
602                })
603                .collect();
604            found.push((
605                "blocked-on-job",
606                format!(
607                    "This cell spent {} s waiting in {}. You could have ended your turn instead: completions arrive as <runtime_event> messages that start a new turn, so wait only when you need the result now.",
608                    jsrt::number::round(waited_ms as f64 / 1000.0) as i64,
609                    calls.join(", ")
610                ),
611            ));
612        }
613        let repeated = cell
614            .status_calls
615            .iter()
616            .find(|(_, count)| *count >= POLL_REPEATS)
617            .map(|(k, _)| k.clone());
618        let again = cell
619            .status_calls
620            .iter()
621            .map(|(k, _)| k)
622            .find(|k| state.previous_status.contains(*k))
623            .cloned();
624        let sleep_loop =
625            matches(&SLEEP, &cell.code) && matches(&LOOP, &cell.code) && matches(&STATUS_CHECK, &cell.code);
626        if repeated.is_some() || again.is_some() || sleep_loop {
627            let what = repeated.or(again);
628            let lead = match what {
629                Some(what) => format!("Repeated status checks ({})", what.replacen(':', " ", 1)),
630                None => "Sleeping in a loop around a status check".into(),
631            };
632            let advice = if events && next_call {
633                "Completions are reported on their own at the top of your next rlm result: do other work instead of checking or sleeping, and when only that result is left, wait once (`await job.result()`, `await rlm.collect(...)`)."
634            } else if events {
635                "Completions arrive as <runtime_event> messages on their own: do other work or end your turn instead of checking or sleeping."
636            } else {
637                "Wait once with `await job.result()`, `await rlm.collect(...)` or `await agents.result(id)` instead of checking and sleeping."
638            };
639            found.push(("poll-loop", format!("{lead} look like polling. {advice}")));
640        }
641        if let Some((ename, count)) = &state.failure
642            && *count >= FAILURE_STREAK
643        {
644            found.push((
645                "repeated-failure",
646                format!(
647                    "{ename} {count} cells in a row. Read the traceback carefully (the failing line and the message) and change your approach instead of retrying a variant of the same cell."
648                ),
649            ));
650        }
651        let truncated = matches(&TRUNCATION_MARKER, outcome.text);
652        let big_string_chars = BIG_STRING
653            .captures(outcome.text)
654            .ok()
655            .flatten()
656            .and_then(|c| c.get(1).map(|m| m.as_str().replace(',', "")))
657            .and_then(|digits| digits.parse::<u64>().ok())
658            .unwrap_or(0);
659        let handle_bytes = self.options.read_handle_bytes;
660        if matches(&FILE_READ, &cell.code) && (truncated || big_string_chars > handle_bytes) {
661            found.push((
662                "large-read",
663                format!(
664                    "This cell read a large file into a string. `await read(path)` returns a ContextHandle for files over {} KiB: use h.search(regex), h.lines(a, b), h.chunks(n) or rlm.map(task, h.chunks(n)) and print only what you need.",
665                    jsrt::number::round(handle_bytes as f64 / 1024.0) as i64
666                ),
667            ));
668        }
669        if truncated {
670            found.push((
671                "output-truncated",
672                "The output was cut in the middle. Keep large data in variables and print slices, counts or matches, or load big text as a handle (`h = await rlm.load(text_or_path)`; `h.search`, `h.lines`).".into(),
673            ));
674        }
675        if let Some(repeat) = repeat {
676            found.push(repeat);
677        }
678        found
679    }
680}
681
682/// Total length of possibly overlapping intervals (waits run concurrently under asyncio.gather).
683fn union_ms(intervals: &[(i64, i64)]) -> i64 {
684    let mut sorted = intervals.to_vec();
685    sorted.sort_by_key(|(start, _)| *start);
686    let mut total = 0;
687    let mut end = i64::MIN;
688    for (start, stop) in sorted {
689        if stop <= end {
690            continue;
691        }
692        total += stop - start.max(end);
693        end = stop;
694    }
695    total
696}