1use crate::hints::HintsModule;
10use crate::util::{Clock, system_clock};
11use fancy_regex::Regex;
12use serde_json::{Map, Value};
13use std::collections::{HashMap, HashSet};
14use std::sync::{Arc, LazyLock, Mutex};
15
16pub const HINTS_ENV: &str = "ULTRON_HINTS";
17pub const HINTS_MAX_ENV: &str = "ULTRON_HINTS_MAX";
18
19pub const DEFAULT_HINT_MAX_PER_TAG: u64 = 3;
20pub const DEFAULT_BLOCKED_SECONDS: f64 = 60.0;
21pub const DEFAULT_READ_HANDLE_BYTES: u64 = 256 * 1024;
22const FAILURE_STREAK: usize = 3;
24const CELL_REPEATS: usize = 3;
26const GUARD_TAGS: &[&str] = &["stuck-loop", "repeated-cell"];
28const POLL_REPEATS: usize = 3;
30
31const BLOCKING_WAITS: &[&str] = &["shell.result", "background.result"];
33
34const STATUS_CALLS: &[&str] = &[
36 "shell.get",
37 "shell.result",
38 "shell.list",
39 "background.inspect",
40 "background.list",
41 "agents.inspect",
42 "agents.tasks",
43 "agents.status",
44 "rlm.list_subagents",
45];
46
47fn regex(pattern: &str) -> Regex {
48 Regex::new(pattern).expect("valid hint pattern")
49}
50
51static TRUNCATION_MARKER: LazyLock<Regex> =
52 LazyLock::new(|| regex(r"\[\.\.\. \d+ bytes truncated(?:; [^\n]*?)? \.\.\.\]"));
53static FILE_READ: LazyLock<Regex> =
54 LazyLock::new(|| regex(r"\.read\(\s*\)|\.read_text\(|\.read_bytes\(|\.readlines\(|\bopen\("));
55static BIG_STRING: LazyLock<Regex> = LazyLock::new(|| regex(r"(?m)^<str: ([\d,]+) chars"));
56static SLEEP: LazyLock<Regex> = LazyLock::new(|| regex(r"\b(?:time|asyncio)\.sleep\("));
57static LOOP: LazyLock<Regex> = LazyLock::new(|| regex(r"(?m)^\s*(?:while|for|async\s+for)\b"));
58static STATUS_CHECK: LazyLock<Regex> = LazyLock::new(|| {
59 regex(
60 r"\.running\b|\.status\b|\.done\(|\.result\(|\.inspect\(|rlm\.jobs\(|rlm\.job\(|list_subagents\(|agents\.tasks\(|agents\.status\(|background\.list\(",
61 )
62});
63static QUERY_COMMANDS: LazyLock<Regex> = LazyLock::new(|| regex(r"^\s*(?:grep|egrep|fgrep|rg|diff|cmp|test|\[)\b"));
64
65fn matches(re: &Regex, text: &str) -> bool {
66 re.is_match(text).unwrap_or(false)
67}
68
69pub fn hints_enabled(value: Option<&str>) -> bool {
71 value.is_some_and(|v| ["on", "1", "true", "yes"].contains(&v.trim().to_lowercase().as_str()))
72}
73
74pub fn hint_max_per_tag(value: Option<&str>) -> u64 {
76 value
77 .map(str::trim)
78 .filter(|v| !v.is_empty())
79 .and_then(|v| v.parse::<u64>().ok())
80 .unwrap_or(DEFAULT_HINT_MAX_PER_TAG)
81}
82
83pub fn read_handle_bytes(value: Option<&str>) -> u64 {
85 value
86 .and_then(|v| v.trim().parse::<u64>().ok())
87 .filter(|v| *v > 0)
88 .unwrap_or(DEFAULT_READ_HANDLE_BYTES)
89}
90
91pub fn bash_failed(command: &str, exit_code: &Value, timed_out: bool) -> bool {
93 if timed_out {
94 return true;
95 }
96 match exit_code.as_f64() {
97 None => false,
98 Some(0.0) => false,
99 Some(code) => !(code == 1.0 && matches(&QUERY_COMMANDS, command)),
100 }
101}
102
103#[derive(Clone, Copy, Debug, PartialEq)]
105pub struct LoopLimits {
106 pub consecutive: usize,
108 pub alternation: usize,
110 pub total: usize,
112}
113
114impl Default for LoopLimits {
115 fn default() -> Self {
116 Self {
117 consecutive: 3,
118 alternation: 4,
119 total: 8,
120 }
121 }
122}
123
124#[derive(Clone, Debug, PartialEq)]
125pub struct LoopTrip {
126 pub level: u8,
127 pub reason: &'static str,
129 pub count: usize,
130 pub ename: Option<String>,
131}
132
133#[derive(Default)]
134struct LaneLoop {
135 history: Vec<(String, bool)>,
136 consecutive: usize,
137 streak_ename: Option<String>,
138 total: usize,
139 level: u8,
140 total_level: u8,
141}
142
143pub fn normalize_cell(code: &str) -> String {
145 code.chars()
146 .filter(|c| !c.is_whitespace() && !c.is_ascii_digit())
147 .collect()
148}
149
150pub fn alternation_length(hashes: &[&str]) -> usize {
152 let n = hashes.len();
153 if n < 2 || hashes[n - 1] == hashes[n - 2] {
154 return 0;
155 }
156 let mut length = 2;
157 let mut i = n as isize - 3;
158 while i >= 0 && hashes[i as usize] == hashes[i as usize + 2] {
159 length += 1;
160 i -= 1;
161 }
162 length
163}
164
165#[derive(Default)]
167pub struct LoopDetector {
168 limits: LoopLimits,
169 lanes: HashMap<String, LaneLoop>,
170}
171
172impl LoopDetector {
173 pub fn new(limits: LoopLimits) -> Self {
174 Self {
175 limits,
176 lanes: HashMap::new(),
177 }
178 }
179
180 pub fn level(&self, lane: &str) -> u8 {
181 self.lanes.get(lane).map_or(0, |s| s.level)
182 }
183
184 pub fn cell_ended(&mut self, lane: &str, code: &str, failed: bool, ename: Option<&str>) -> Option<LoopTrip> {
186 let limits = self.limits;
187 let state = self.lanes.entry(lane.to_string()).or_default();
188 let hash = jsrt::hash::sha256_hex(normalize_cell(code).as_bytes())[..16].to_string();
189 state.history.push((hash.clone(), failed));
190 if state.history.len() > limits.alternation * 2 + 2 {
191 state.history.remove(0);
192 }
193 let ename = ename.unwrap_or("").to_string();
194 if failed {
195 state.consecutive += 1;
196 state.total += 1;
197 state.streak_ename = Some(
198 if state.consecutive == 1 || state.streak_ename.as_deref() == Some(&ename) {
199 ename
200 } else {
201 String::new()
202 },
203 );
204 } else {
205 state.consecutive = 0;
206 state.streak_ename = None;
207 }
208 let n = state.history.len();
209 let repeat = n >= 3 && state.history[n - 3].0 == hash;
210 if !failed && !repeat {
212 state.level = 0;
213 }
214 let hashes: Vec<&str> = state.history.iter().map(|(h, _)| h.as_str()).collect();
215 let run = alternation_length(&hashes);
216 let run_failures = state.history[n - run..].iter().filter(|(_, f)| *f).count();
217 let alternating = run >= limits.alternation && run_failures * 2 >= run;
218 let trip = |state: &mut LaneLoop, level: u8, reason: &'static str, count: usize| {
219 state.level = level;
220 if reason == "total" {
221 state.total_level = level;
222 }
223 let ename = (reason == "consecutive")
224 .then(|| state.streak_ename.clone().filter(|e| !e.is_empty()))
225 .flatten();
226 Some(LoopTrip {
227 level,
228 reason,
229 count,
230 ename,
231 })
232 };
233 if state.level == 0 {
234 if limits.consecutive > 0 && state.consecutive >= limits.consecutive {
235 let count = state.consecutive;
236 return trip(state, 1, "consecutive", count);
237 }
238 if limits.alternation > 0 && alternating {
239 return trip(state, 1, "alternation", run);
240 }
241 } else if state.level == 1 {
242 if limits.consecutive > 0 && state.consecutive >= limits.consecutive * 2 {
243 let count = state.consecutive;
244 return trip(state, 2, "consecutive", count);
245 }
246 if limits.alternation > 0 && alternating && run >= limits.alternation * 2 {
247 return trip(state, 2, "alternation", run);
248 }
249 }
250 if limits.total > 0 && failed {
252 let total = state.total;
253 if state.total_level == 0 && state.level == 0 && total >= limits.total {
254 return trip(state, 1, "total", total);
255 }
256 if state.total_level == 1 && state.level < 2 && total >= limits.total * 2 {
257 return trip(state, 2, "total", total);
258 }
259 }
260 None
261 }
262
263 pub fn reset(&mut self, lane: &str) {
264 self.lanes.remove(lane);
265 }
266}
267
268pub fn loop_hint_text(trip: &LoopTrip) -> String {
270 let what = match trip.reason {
271 "consecutive" => format!(
272 "The last {} cells failed{}",
273 trip.count,
274 trip.ename
275 .as_ref()
276 .map_or(String::new(), |e| format!(" ({e} each time)"))
277 ),
278 "alternation" => format!(
279 "The last {} cells alternate between two near-identical versions",
280 trip.count
281 ),
282 _ => format!("{} cells have failed in this run", trip.count),
283 };
284 if trip.level == 1 {
285 format!(
286 "{what}; you look stuck. Before the next cell, state three different hypotheses for why it fails, then run one cell that tests the most likely one instead of retrying a variant."
287 )
288 } else {
289 format!(
290 "{what}, even after the last hint. Stop repeating this approach: name three hypotheses you have not tested yet and test the most likely; if none holds, say what blocks you."
291 )
292 }
293}
294
295#[derive(Clone)]
296pub struct CellHintsOptions {
297 pub enabled: bool,
298 pub guards: bool,
301 pub max_per_tag: u64,
302 pub async_events: bool,
304 pub next_call_root: bool,
307 pub blocked_seconds: f64,
308 pub read_handle_bytes: u64,
309 pub loop_limits: LoopLimits,
310 pub now: Clock,
311}
312
313impl Default for CellHintsOptions {
314 fn default() -> Self {
315 Self {
316 enabled: true,
317 guards: true,
318 max_per_tag: DEFAULT_HINT_MAX_PER_TAG,
319 async_events: true,
320 next_call_root: false,
321 blocked_seconds: DEFAULT_BLOCKED_SECONDS,
322 read_handle_bytes: DEFAULT_READ_HANDLE_BYTES,
323 loop_limits: LoopLimits::default(),
324 now: system_clock(),
325 }
326 }
327}
328
329pub struct CellOutcome<'a> {
331 pub text: &'a str,
333 pub ename: Option<&'a str>,
335}
336
337#[derive(Default)]
338struct Cell {
339 code: String,
340 detached: Vec<(String, Option<f64>)>,
342 waits: Vec<(i64, i64)>,
343 wait_types: Vec<String>,
344 status_calls: Vec<(String, usize)>,
346 bash_failed: bool,
347}
348
349#[derive(Default)]
350struct LaneState {
351 cell: Option<Cell>,
352 previous_status: HashSet<String>,
353 failure: Option<(String, usize)>,
354 repeat: Option<(String, usize)>,
355}
356
357pub type LaneWoken = Arc<dyn Fn(&str) -> bool + Send + Sync>;
359
360pub struct CellHints {
361 module: Arc<HintsModule>,
362 options: CellHintsOptions,
363 lanes: Mutex<HashMap<String, LaneState>>,
364 loops: Mutex<LoopDetector>,
365 woken: Mutex<Option<LaneWoken>>,
366}
367
368impl CellHints {
369 pub fn new(module: Arc<HintsModule>, options: CellHintsOptions) -> Self {
370 let loops = LoopDetector::new(options.loop_limits);
371 Self {
372 module,
373 options,
374 lanes: Mutex::new(HashMap::new()),
375 loops: Mutex::new(loops),
376 woken: Mutex::new(None),
377 }
378 }
379
380 pub fn set_lane_woken(&self, woken: LaneWoken) {
383 *jsrt::sync::lock(&self.woken) = Some(woken);
384 }
385
386 fn woken(&self, lane: &str) -> bool {
388 let woken = jsrt::sync::lock(&self.woken).clone();
389 match woken {
390 Some(woken) => woken(lane),
391 None => lane == kernel::ROOT_LANE && !self.options.next_call_root,
392 }
393 }
394
395 pub fn enabled(&self) -> bool {
396 self.options.enabled
397 }
398
399 pub fn active(&self) -> bool {
401 self.options.enabled || self.options.guards
402 }
403
404 pub fn refusal(&self, lane: &str, code: &str) -> Option<String> {
407 if !self.options.guards {
408 return None;
409 }
410 let lanes = jsrt::sync::lock(&self.lanes);
411 let (previous, count) = lanes.get(lane)?.repeat.as_ref()?;
412 (*count >= CELL_REPEATS * 2 && previous == code.trim()).then(|| {
413 format!(
414 "Not run: this exact cell already ran {count} times in a row and succeeded each time. Use the output you have, run something different, or end your turn and wait for the event."
415 )
416 })
417 }
418
419 fn now(&self) -> i64 {
420 (self.options.now)()
421 }
422
423 pub fn run_ended(&self, lane: &str) {
425 jsrt::sync::lock(&self.loops).reset(lane);
426 if let Some(state) = jsrt::sync::lock(&self.lanes).get_mut(lane) {
427 state.failure = None;
428 state.repeat = None;
429 }
430 }
431
432 pub fn begin_cell(&self, lane: &str, code: &str) {
434 if !self.active() {
435 return;
436 }
437 jsrt::sync::lock(&self.lanes).entry(lane.to_string()).or_default().cell = Some(Cell {
438 code: code.to_string(),
439 ..Cell::default()
440 });
441 }
442
443 pub fn observe(
445 &self,
446 lane: &str,
447 kind: &str,
448 payload: &Map<String, Value>,
449 result: Option<&Value>,
450 started_at: i64,
451 ) {
452 let now = self.now();
453 let mut lanes = jsrt::sync::lock(&self.lanes);
454 let Some(cell) = lanes.get_mut(lane).and_then(|s| s.cell.as_mut()) else {
455 return;
456 };
457 let reply = result.and_then(Value::as_object);
458 let running = reply.is_some_and(|r| r.get("running") == Some(&Value::Bool(true)));
459 if kind == "bash"
460 && let Some(reply) = reply
461 && !running
462 {
463 cell.bash_failed = bash_failed(
464 payload.get("command").and_then(Value::as_str).unwrap_or(""),
465 reply.get("exit_code").unwrap_or(&Value::Null),
466 reply.get("timed_out") == Some(&Value::Bool(true)),
467 );
468 }
469 if kind == "bash" && running {
470 let id = reply
471 .and_then(|r| r.get("job"))
472 .and_then(|j| j.get("id"))
473 .and_then(Value::as_str)
474 .unwrap_or("a job");
475 let waited = payload.get("yield_after").and_then(Value::as_f64);
476 cell.detached.push((id.to_string(), waited));
477 }
478 if BLOCKING_WAITS.contains(&kind) {
479 cell.waits.push((started_at, now));
480 if !cell.wait_types.iter().any(|t| t == kind) {
481 cell.wait_types.push(kind.to_string());
482 }
483 }
484 if STATUS_CALLS.contains(&kind) {
485 if (kind == "shell.get" || kind == "shell.result") && !running {
487 return;
488 }
489 let key = match payload.get("id").and_then(Value::as_str) {
490 Some(id) => format!("{kind}:{id}"),
491 None => kind.to_string(),
492 };
493 match cell.status_calls.iter_mut().find(|(k, _)| *k == key) {
494 Some(entry) => entry.1 += 1,
495 None => cell.status_calls.push((key, 1)),
496 }
497 }
498 }
499
500 pub fn end_cell(&self, lane: &str, outcome: &CellOutcome<'_>) -> Option<String> {
502 if !self.active() {
503 return None;
504 }
505 let next_call = !self.woken(lane);
507 let (mut candidates, code, failed) = {
508 let mut lanes = jsrt::sync::lock(&self.lanes);
509 let state = lanes.entry(lane.to_string()).or_default();
510 let cell = state.cell.take()?;
511 let candidates = self.candidates(state, &cell, outcome, next_call);
512 state.previous_status = cell.status_calls.iter().map(|(k, _)| k.clone()).collect();
513 (candidates, cell.code, outcome.ename.is_some() || cell.bash_failed)
514 };
515 let (trip, level) = {
516 let mut loops = jsrt::sync::lock(&self.loops);
517 let trip = loops.cell_ended(lane, &code, failed, outcome.ename);
518 (trip, loops.level(lane))
519 };
520 if let Some(trip) = trip {
521 candidates.insert(0, ("stuck-loop", loop_hint_text(&trip)));
522 }
523 if !self.options.enabled {
524 candidates.retain(|(tag, _)| GUARD_TAGS.contains(tag));
525 }
526 if candidates.is_empty() {
527 return None;
528 }
529 self.module.fire(lane, &candidates, self.options.max_per_tag, level > 0)
530 }
531
532 fn candidates(
534 &self,
535 state: &mut LaneState,
536 cell: &Cell,
537 outcome: &CellOutcome<'_>,
538 next_call: bool,
539 ) -> Vec<(&'static str, String)> {
540 let events = self.options.async_events;
541 let mut found: Vec<(&'static str, String)> = Vec::new();
542 state.failure = match (outcome.ename, state.failure.take()) {
544 (Some(ename), Some((previous, count))) if previous == ename => Some((previous, count + 1)),
545 (Some(ename), _) => Some((ename.to_string(), 1)),
546 (None, _) => None,
547 };
548 let code = cell.code.trim().to_string();
550 state.repeat = if outcome.ename.is_none() && !cell.bash_failed {
551 match state.repeat.take() {
552 Some((previous, count)) if previous == code => Some((code, count + 1)),
553 _ => Some((code, 1)),
554 }
555 } else {
556 None
557 };
558 let repeat = state
559 .repeat
560 .as_ref()
561 .filter(|(_, count)| *count == CELL_REPEATS || *count == CELL_REPEATS * 2)
562 .map(|(_, count)| {
563 (
564 "repeated-cell",
565 format!(
566 "This exact cell has now run {count} times in a row and succeeded each time; rerunning it will not tell you more. Use the output you have, keep values in variables instead of recomputing them, and wait for running work once (`await job.result()`, `await rlm.collect(...)`) instead of checking again."
567 ),
568 )
569 });
570
571 if let Some((id, waited)) = cell.detached.first() {
572 let after = waited.map_or(String::new(), |w| format!(" after {} s", jsrt::number::to_string(w)));
573 found.push((
574 "job-detached",
575 if events && next_call {
576 format!(
577 "The command was still running{after}, so it continues as job {id}. Do other work meanwhile: its completion is reported as a <runtime_event> at the top of a later rlm result, and `await <result>.job.result()` waits for it when your answer needs it (nothing wakes you after you reply). (`yield_after=None` blocks until a command ends; `yield_after=0` detaches at once.)"
578 )
579 } else if events {
580 format!(
581 "The command was still running{after}, so it continues as job {id}. Do not wait for it: do other work or end your turn; its completion arrives as a <runtime_event> that starts your next turn. (`yield_after=None` blocks until a command ends; `yield_after=0` detaches at once.)"
582 )
583 } else {
584 format!(
585 "The command was still running{after}, so it continues as job {id}; `await <result>.job.result()` waits for it. (`yield_after=None` blocks until a command ends.)"
586 )
587 },
588 ));
589 }
590 let waited_ms = union_ms(&cell.waits);
591 if events && !next_call && waited_ms as f64 >= self.options.blocked_seconds * 1000.0 {
593 let calls: Vec<&str> = cell
594 .wait_types
595 .iter()
596 .map(|t| {
597 if t == "shell.result" {
598 "job.result()"
599 } else {
600 "background.result()"
601 }
602 })
603 .collect();
604 found.push((
605 "blocked-on-job",
606 format!(
607 "This cell spent {} s waiting in {}. You could have ended your turn instead: completions arrive as <runtime_event> messages that start a new turn, so wait only when you need the result now.",
608 jsrt::number::round(waited_ms as f64 / 1000.0) as i64,
609 calls.join(", ")
610 ),
611 ));
612 }
613 let repeated = cell
614 .status_calls
615 .iter()
616 .find(|(_, count)| *count >= POLL_REPEATS)
617 .map(|(k, _)| k.clone());
618 let again = cell
619 .status_calls
620 .iter()
621 .map(|(k, _)| k)
622 .find(|k| state.previous_status.contains(*k))
623 .cloned();
624 let sleep_loop =
625 matches(&SLEEP, &cell.code) && matches(&LOOP, &cell.code) && matches(&STATUS_CHECK, &cell.code);
626 if repeated.is_some() || again.is_some() || sleep_loop {
627 let what = repeated.or(again);
628 let lead = match what {
629 Some(what) => format!("Repeated status checks ({})", what.replacen(':', " ", 1)),
630 None => "Sleeping in a loop around a status check".into(),
631 };
632 let advice = if events && next_call {
633 "Completions are reported on their own at the top of your next rlm result: do other work instead of checking or sleeping, and when only that result is left, wait once (`await job.result()`, `await rlm.collect(...)`)."
634 } else if events {
635 "Completions arrive as <runtime_event> messages on their own: do other work or end your turn instead of checking or sleeping."
636 } else {
637 "Wait once with `await job.result()`, `await rlm.collect(...)` or `await agents.result(id)` instead of checking and sleeping."
638 };
639 found.push(("poll-loop", format!("{lead} look like polling. {advice}")));
640 }
641 if let Some((ename, count)) = &state.failure
642 && *count >= FAILURE_STREAK
643 {
644 found.push((
645 "repeated-failure",
646 format!(
647 "{ename} {count} cells in a row. Read the traceback carefully (the failing line and the message) and change your approach instead of retrying a variant of the same cell."
648 ),
649 ));
650 }
651 let truncated = matches(&TRUNCATION_MARKER, outcome.text);
652 let big_string_chars = BIG_STRING
653 .captures(outcome.text)
654 .ok()
655 .flatten()
656 .and_then(|c| c.get(1).map(|m| m.as_str().replace(',', "")))
657 .and_then(|digits| digits.parse::<u64>().ok())
658 .unwrap_or(0);
659 let handle_bytes = self.options.read_handle_bytes;
660 if matches(&FILE_READ, &cell.code) && (truncated || big_string_chars > handle_bytes) {
661 found.push((
662 "large-read",
663 format!(
664 "This cell read a large file into a string. `await read(path)` returns a ContextHandle for files over {} KiB: use h.search(regex), h.lines(a, b), h.chunks(n) or rlm.map(task, h.chunks(n)) and print only what you need.",
665 jsrt::number::round(handle_bytes as f64 / 1024.0) as i64
666 ),
667 ));
668 }
669 if truncated {
670 found.push((
671 "output-truncated",
672 "The output was cut in the middle. Keep large data in variables and print slices, counts or matches, or load big text as a handle (`h = await rlm.load(text_or_path)`; `h.search`, `h.lines`).".into(),
673 ));
674 }
675 if let Some(repeat) = repeat {
676 found.push(repeat);
677 }
678 found
679 }
680}
681
682fn union_ms(intervals: &[(i64, i64)]) -> i64 {
684 let mut sorted = intervals.to_vec();
685 sorted.sort_by_key(|(start, _)| *start);
686 let mut total = 0;
687 let mut end = i64::MIN;
688 for (start, stop) in sorted {
689 if stop <= end {
690 continue;
691 }
692 total += stop - start.max(end);
693 end = stop;
694 }
695 total
696}