Skip to main content

reflex/pulse/write/
run.rs

1//! The writing-pass executor: plan, budget, call, cache, degrade.
2//!
3//! Flow of [`run_tasks`]:
4//! 1. **Plan.** Suppress tasks with too little input. Compute each cache key. A hit is
5//!    used unless the task is in the `--force-renarrate` scope. Tasks with the same key
6//!    share one call.
7//! 2. **Budget.** With `max_llm_tokens`, keep calls in `(priority, id)` order until the
8//!    estimate reaches the cap; the rest are deferred. The cache fills, so later runs
9//!    complete the rest.
10//! 3. **Dry run** stops here.
11//! 4. **Probe.** The first call runs alone. A fatal error (auth, unknown model, bad
12//!    request) degrades the whole run to structural output.
13//! 5. **Fan-out** under a semaphore, with retries and backoff for transient errors,
14//!    halved concurrency after a 429, and a circuit breaker after consecutive failures.
15//! 6. **Verdict.** If fewer than `degrade_threshold` of the eligible tasks succeeded, the
16//!    whole run degrades. Successful answers stay cached either way.
17//! 7. **Prune** the cache after a complete, successful run.
18
19use super::cache::{CacheEntry, KeyMaterial, WriteCache};
20use super::provider::LlmHandle;
21use super::{ForceScope, LlmMode};
22use crate::semantic::providers::{
23    CompletionRequest, LlmProvider, OutputMode, ProviderError, ProviderErrorKind, StopReason, Usage,
24};
25use std::collections::BTreeMap;
26use std::fmt::Write as _;
27use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
28use std::sync::{Arc, Mutex};
29use std::time::{Duration, Instant};
30
31/// The output contract of one task.
32#[derive(Debug, Clone, PartialEq)]
33pub enum OutputSpec {
34    Text,
35    JsonObject,
36    JsonSchema {
37        name: String,
38        schema: serde_json::Value,
39    },
40}
41
42impl OutputSpec {
43    pub fn as_mode(&self) -> OutputMode<'_> {
44        match self {
45            OutputSpec::Text => OutputMode::Text,
46            OutputSpec::JsonObject => OutputMode::JsonObject,
47            OutputSpec::JsonSchema { name, schema } => OutputMode::JsonSchema { name, schema },
48        }
49    }
50}
51
52/// One unit of LLM writing.
53#[derive(Debug, Clone)]
54pub struct WriteTask {
55    /// Unique and stable, e.g. `module:src/pulse`. Results are keyed by it.
56    pub id: String,
57    /// Task kind, used for `--force-renarrate` scopes and the plan table.
58    pub kind: &'static str,
59    /// Bump when parsing or prompt semantics change without a text change.
60    pub prompt_version: u32,
61    pub system: String,
62    pub user: String,
63    pub output: OutputSpec,
64    pub max_tokens: u32,
65    /// Lower runs first when a budget truncates the plan.
66    pub priority: u16,
67    /// Suppress the task when `user` has fewer words than this.
68    pub min_words: usize,
69}
70
71impl WriteTask {
72    /// A free-text task with default limits.
73    pub fn text(
74        id: impl Into<String>,
75        kind: &'static str,
76        system: impl Into<String>,
77        user: impl Into<String>,
78    ) -> Self {
79        Self {
80            id: id.into(),
81            kind,
82            prompt_version: 1,
83            system: system.into(),
84            user: user.into(),
85            output: OutputSpec::Text,
86            max_tokens: 1200,
87            priority: 100,
88            min_words: 15,
89        }
90    }
91
92    pub fn with_output(mut self, output: OutputSpec) -> Self {
93        self.output = output;
94        self
95    }
96
97    pub fn with_max_tokens(mut self, max_tokens: u32) -> Self {
98        self.max_tokens = max_tokens;
99        self
100    }
101
102    pub fn with_priority(mut self, priority: u16) -> Self {
103        self.priority = priority;
104        self
105    }
106
107    pub fn with_prompt_version(mut self, v: u32) -> Self {
108        self.prompt_version = v;
109        self
110    }
111
112    fn est_input_tokens(&self) -> u64 {
113        // ~3.5 characters per token; the budget headroom absorbs the error.
114        ((self.system.len() + self.user.len()) as f64 / 3.5).ceil() as u64
115    }
116
117    fn est_output_tokens(&self) -> u64 {
118        (self.max_tokens as f64 * 0.6).ceil() as u64
119    }
120}
121
122/// Executor settings.
123#[derive(Debug, Clone)]
124pub struct WriteConfig {
125    pub mode: LlmMode,
126    /// Concurrent calls. `0` means one per job.
127    pub concurrency: usize,
128    pub max_llm_tokens: Option<u64>,
129    pub force: ForceScope,
130    pub dry_run: bool,
131    /// Minimum share of eligible tasks that must succeed, else the run degrades.
132    pub degrade_threshold: f32,
133    pub keep_runs: usize,
134    pub prune: bool,
135    pub cache_model_agnostic: bool,
136    /// Attempts per call, including the first.
137    pub max_attempts: u32,
138    pub base_backoff: Duration,
139    /// Consecutive failed calls that stop the run.
140    pub breaker_threshold: usize,
141}
142
143impl Default for WriteConfig {
144    fn default() -> Self {
145        Self {
146            mode: LlmMode::On,
147            concurrency: 4,
148            max_llm_tokens: None,
149            force: ForceScope::None,
150            dry_run: false,
151            degrade_threshold: 0.9,
152            keep_runs: 3,
153            prune: true,
154            cache_model_agnostic: false,
155            max_attempts: 3,
156            base_backoff: Duration::from_secs(1),
157            breaker_threshold: 5,
158        }
159    }
160}
161
162#[derive(Debug, Clone, PartialEq)]
163pub enum TaskOutcome {
164    /// Answer read from the cache.
165    Cached,
166    /// Answer from a provider call this run.
167    Called,
168    /// Not sent: the input is too thin to write about.
169    Suppressed(String),
170    /// Not sent: outside the token budget or the run stopped first.
171    Deferred,
172    /// Cache-only mode and no entry.
173    NotCached,
174    Failed(String),
175    /// The LLM is off.
176    Skipped,
177}
178
179#[derive(Debug, Clone)]
180pub struct TaskResult {
181    pub kind: &'static str,
182    pub outcome: TaskOutcome,
183    pub text: Option<String>,
184    pub usage: Option<Usage>,
185    est_input: u64,
186    est_output: u64,
187}
188
189#[derive(Debug, Clone, Default)]
190pub struct RunStats {
191    pub cached: usize,
192    pub called: usize,
193    pub suppressed: usize,
194    pub deferred: usize,
195    pub not_cached: usize,
196    pub failed: usize,
197    pub skipped: usize,
198    pub calls_made: usize,
199    pub retries: usize,
200    pub input_tokens: u64,
201    pub output_tokens: u64,
202    pub cached_input_tokens: u64,
203    pub pruned: usize,
204    pub elapsed: Duration,
205}
206
207/// Everything a run produced.
208#[derive(Debug, Clone)]
209pub struct WriteOutcome {
210    pub mode: LlmMode,
211    pub results: BTreeMap<String, TaskResult>,
212    /// Set when the whole run fell back to structural output.
213    pub degraded: Option<String>,
214    /// Some calls were deferred by the token budget.
215    pub truncated: bool,
216    pub dry_run: bool,
217    pub model: Option<String>,
218    pub stats: RunStats,
219}
220
221impl WriteOutcome {
222    /// The text for a task, or `None` when the section must render structurally.
223    pub fn text(&self, id: &str) -> Option<&str> {
224        if self.degraded.is_some() || self.dry_run {
225            return None;
226        }
227        self.results.get(id)?.text.as_deref()
228    }
229
230    /// True when any task produced text.
231    pub fn has_text(&self) -> bool {
232        self.degraded.is_none() && !self.dry_run && self.results.values().any(|r| r.text.is_some())
233    }
234
235    /// Per-kind plan: tasks, cached, to-call, suppressed, deferred, estimated tokens.
236    pub fn plan_table(&self) -> String {
237        #[derive(Default)]
238        struct Row {
239            tasks: usize,
240            cached: usize,
241            to_call: usize,
242            suppressed: usize,
243            deferred: usize,
244            est_in: u64,
245            est_out: u64,
246        }
247        let mut rows: BTreeMap<&str, Row> = BTreeMap::new();
248        let mut total = Row::default();
249        for r in self.results.values() {
250            for row in [rows.entry(r.kind).or_default(), &mut total] {
251                row.tasks += 1;
252                match r.outcome {
253                    TaskOutcome::Cached => row.cached += 1,
254                    TaskOutcome::Suppressed(_) => row.suppressed += 1,
255                    TaskOutcome::Deferred => row.deferred += 1,
256                    TaskOutcome::Called | TaskOutcome::Failed(_) => {
257                        row.to_call += 1;
258                        row.est_in += r.est_input;
259                        row.est_out += r.est_output;
260                    }
261                    TaskOutcome::NotCached | TaskOutcome::Skipped => {}
262                }
263            }
264        }
265        let mut out = String::new();
266        let _ = writeln!(
267            out,
268            "{:<14}{:>7}{:>8}{:>9}{:>12}{:>10}{:>10}{:>10}",
269            "kind", "tasks", "cached", "to-call", "suppressed", "deferred", "est.in", "est.out"
270        );
271        let mut line = |name: &str, r: &Row| {
272            let _ = writeln!(
273                out,
274                "{:<14}{:>7}{:>8}{:>9}{:>12}{:>10}{:>10}{:>10}",
275                name,
276                r.tasks,
277                r.cached,
278                r.to_call,
279                r.suppressed,
280                r.deferred,
281                human_tokens(r.est_in),
282                human_tokens(r.est_out)
283            );
284        };
285        for (kind, r) in &rows {
286            line(kind, r);
287        }
288        line("total", &total);
289        if let Some(model) = &self.model {
290            let _ = writeln!(out, "model: {model}");
291        }
292        out
293    }
294
295    /// One-line run summary for stderr.
296    pub fn summary(&self) -> String {
297        let s = &self.stats;
298        let mut line = format!(
299            "LLM: {} cached, {} called, {} suppressed",
300            s.cached, s.called, s.suppressed
301        );
302        for (n, label) in [
303            (s.deferred, "deferred"),
304            (s.not_cached, "not cached"),
305            (s.failed, "failed"),
306        ] {
307            if n > 0 {
308                let _ = write!(line, ", {n} {label}");
309            }
310        }
311        if s.calls_made > 0 {
312            let _ = write!(
313                line,
314                "; tokens in {} (cached {}) / out {}",
315                human_tokens(s.input_tokens),
316                human_tokens(s.cached_input_tokens),
317                human_tokens(s.output_tokens)
318            );
319        }
320        if s.retries > 0 {
321            let _ = write!(line, "; {} retries", s.retries);
322        }
323        if s.pruned > 0 {
324            let _ = write!(line, "; pruned {} cache entries", s.pruned);
325        }
326        let _ = write!(line, " ({:.1}s)", s.elapsed.as_secs_f64());
327        if let Some(reason) = &self.degraded {
328            let _ = write!(line, "\nLLM output disabled for this run: {reason}");
329        }
330        line
331    }
332}
333
334fn human_tokens(n: u64) -> String {
335    if n >= 10_000 {
336        format!("{:.1}k", n as f64 / 1000.0)
337    } else {
338        n.to_string()
339    }
340}
341
342/// Tasks that share one cache key and therefore one call.
343struct Job {
344    key: String,
345    ids: Vec<String>,
346    task: WriteTask,
347}
348
349struct Shared {
350    sem: tokio::sync::Semaphore,
351    permits: AtomicUsize,
352    consecutive_failures: AtomicUsize,
353    stop: Mutex<Option<String>>,
354    tokens_used: AtomicU64,
355    retries: AtomicUsize,
356    calls: AtomicUsize,
357}
358
359impl Shared {
360    fn stopped(&self) -> bool {
361        self.stop.lock().unwrap().is_some()
362    }
363
364    fn stop_with(&self, reason: String) {
365        let mut stop = self.stop.lock().unwrap();
366        if stop.is_none() {
367            *stop = Some(reason);
368        }
369    }
370
371    /// After a 429, halve the number of concurrent calls (floor 1).
372    fn halve_concurrency(&self) {
373        let current = self.permits.load(Ordering::SeqCst);
374        if current > 1 {
375            let forgotten = self.sem.forget_permits(current / 2);
376            self.permits.fetch_sub(forgotten, Ordering::SeqCst);
377            if forgotten > 0 {
378                log::info!(
379                    "pulse write: rate limited; concurrency {current} → {}",
380                    current - forgotten
381                );
382            }
383        }
384    }
385}
386
387enum CallError {
388    Fatal(String),
389    Failed(String),
390}
391
392type CallResult = Result<(String, Option<Usage>), CallError>;
393
394/// Run the writing pass.
395pub fn run_tasks(
396    mut tasks: Vec<WriteTask>,
397    llm: &LlmHandle,
398    cache: &WriteCache,
399    cfg: &WriteConfig,
400) -> WriteOutcome {
401    let start = Instant::now();
402    tasks.sort_by(|a, b| a.id.cmp(&b.id));
403    tasks.dedup_by(|a, b| a.id == b.id);
404
405    let mut outcome = WriteOutcome {
406        mode: cfg.mode,
407        results: BTreeMap::new(),
408        degraded: None,
409        truncated: false,
410        dry_run: cfg.dry_run,
411        model: llm
412            .ident
413            .as_ref()
414            .map(|i| format!("{}/{}", i.provider, i.model)),
415        stats: RunStats::default(),
416    };
417    let result = |task: &WriteTask, outcome: TaskOutcome| TaskResult {
418        kind: task.kind,
419        outcome,
420        text: None,
421        usage: None,
422        est_input: task.est_input_tokens(),
423        est_output: task.est_output_tokens(),
424    };
425
426    if cfg.mode == LlmMode::Off {
427        for t in &tasks {
428            outcome
429                .results
430                .insert(t.id.clone(), result(t, TaskOutcome::Skipped));
431        }
432        return finish(outcome, start);
433    }
434
435    let Some(ident) = llm.ident.clone() else {
436        let reason = llm
437            .unavailable
438            .clone()
439            .unwrap_or_else(|| "no LLM provider configured".into());
440        for t in &tasks {
441            outcome
442                .results
443                .insert(t.id.clone(), result(t, TaskOutcome::NotCached));
444        }
445        if cfg.mode == LlmMode::On {
446            outcome.degraded = Some(reason);
447        }
448        return finish(outcome, start);
449    };
450
451    // ── 1. Plan ─────────────────────────────────────────────────────────────
452    let mut jobs: BTreeMap<String, Job> = BTreeMap::new();
453    let mut used_keys: Vec<String> = Vec::new();
454    for task in &tasks {
455        let words = task.user.split_whitespace().count();
456        if words < task.min_words {
457            outcome.results.insert(
458                task.id.clone(),
459                result(
460                    task,
461                    TaskOutcome::Suppressed(format!("{words} words of input")),
462                ),
463            );
464            continue;
465        }
466        let schema_json = match &task.output {
467            OutputSpec::JsonSchema { schema, .. } => Some(schema.to_string()),
468            _ => None,
469        };
470        let agnostic = cfg.cache_model_agnostic;
471        let key = KeyMaterial {
472            task_kind: task.kind,
473            prompt_version: task.prompt_version,
474            provider: (!agnostic).then_some(ident.provider.as_str()),
475            model: (!agnostic).then_some(ident.model.as_str()),
476            max_tokens: task.max_tokens,
477            output_mode: task.output.as_mode().label(),
478            output_schema: schema_json.as_deref(),
479            system: &task.system,
480            user: &task.user,
481        }
482        .key();
483
484        if !cfg.force.matches(task)
485            && let Some(hit) = cache.get(&key)
486        {
487            let mut r = result(task, TaskOutcome::Cached);
488            r.text = Some(hit.text);
489            outcome.results.insert(task.id.clone(), r);
490            used_keys.push(key);
491            continue;
492        }
493        if cfg.mode == LlmMode::CacheOnly {
494            outcome
495                .results
496                .insert(task.id.clone(), result(task, TaskOutcome::NotCached));
497            continue;
498        }
499        outcome
500            .results
501            .insert(task.id.clone(), result(task, TaskOutcome::Called));
502        jobs.entry(key.clone())
503            .or_insert_with(|| Job {
504                key,
505                ids: Vec::new(),
506                task: task.clone(),
507            })
508            .ids
509            .push(task.id.clone());
510    }
511
512    // ── 2. Budget ───────────────────────────────────────────────────────────
513    let mut jobs: Vec<Job> = jobs.into_values().collect();
514    jobs.sort_by(|a, b| (a.task.priority, &a.task.id).cmp(&(b.task.priority, &b.task.id)));
515    if let Some(cap) = cfg.max_llm_tokens {
516        let mut spent = 0u64;
517        let mut kept = Vec::with_capacity(jobs.len());
518        for job in jobs {
519            let cost = job.task.est_input_tokens() + job.task.est_output_tokens();
520            if spent + cost <= cap {
521                spent += cost;
522                kept.push(job);
523            } else {
524                outcome.truncated = true;
525                for id in &job.ids {
526                    if let Some(r) = outcome.results.get_mut(id) {
527                        r.outcome = TaskOutcome::Deferred;
528                    }
529                }
530            }
531        }
532        jobs = kept;
533    }
534
535    // ── 3. Dry run ──────────────────────────────────────────────────────────
536    if cfg.dry_run || jobs.is_empty() {
537        return verdict(outcome, used_keys, cache, cfg, start);
538    }
539
540    let Some(provider) = llm.provider.clone() else {
541        outcome.degraded = Some(
542            llm.unavailable
543                .clone()
544                .unwrap_or_else(|| "LLM provider unavailable".into()),
545        );
546        for job in &jobs {
547            for id in &job.ids {
548                if let Some(r) = outcome.results.get_mut(id) {
549                    r.outcome = TaskOutcome::Failed("provider unavailable".into());
550                }
551            }
552        }
553        return finish(outcome, start);
554    };
555
556    // ── 4–5. Probe, then fan out ────────────────────────────────────────────
557    let permits = if cfg.concurrency == 0 {
558        jobs.len()
559    } else {
560        cfg.concurrency
561    }
562    .max(1);
563    let shared = Arc::new(Shared {
564        sem: tokio::sync::Semaphore::new(permits),
565        permits: AtomicUsize::new(permits),
566        consecutive_failures: AtomicUsize::new(0),
567        stop: Mutex::new(None),
568        tokens_used: AtomicU64::new(0),
569        retries: AtomicUsize::new(0),
570        calls: AtomicUsize::new(0),
571    });
572    eprintln!(
573        "  Writing: {} LLM call(s), {} concurrent ({})",
574        jobs.len(),
575        permits,
576        outcome.model.as_deref().unwrap_or("?")
577    );
578
579    let runtime = match tokio::runtime::Builder::new_multi_thread()
580        .worker_threads(2)
581        .enable_all()
582        .build()
583    {
584        Ok(rt) => rt,
585        Err(e) => {
586            outcome.degraded = Some(format!("could not start async runtime: {e}"));
587            return finish(outcome, start);
588        }
589    };
590
591    let cfg_arc = Arc::new(cfg.clone());
592    let cache_arc = Arc::new(cache.clone());
593    let finished: Vec<(Job, CallResult)> = runtime.block_on(async {
594        let mut done = Vec::with_capacity(jobs.len());
595        let mut rest = jobs.into_iter();
596        let probe = rest.next().expect("jobs is non-empty");
597        let probe_result =
598            execute(Arc::clone(&provider), &probe, &cfg_arc, &shared, &cache_arc).await;
599        done.push((probe, probe_result));
600
601        let mut set = tokio::task::JoinSet::new();
602        for job in rest {
603            let provider = Arc::clone(&provider);
604            let cfg = Arc::clone(&cfg_arc);
605            let shared = Arc::clone(&shared);
606            let cache = Arc::clone(&cache_arc);
607            set.spawn(async move {
608                let permit = shared.sem.acquire().await;
609                if permit.is_err() || shared.stopped() {
610                    return (job, None);
611                }
612                if let Some(cap) = cfg.max_llm_tokens
613                    && shared.tokens_used.load(Ordering::SeqCst) >= cap
614                {
615                    return (job, None);
616                }
617                let r = execute(provider, &job, &cfg, &shared, &cache).await;
618                (job, Some(r))
619            });
620        }
621        while let Some(joined) = set.join_next().await {
622            match joined {
623                Ok((job, Some(r))) => done.push((job, r)),
624                Ok((job, None)) => {
625                    for id in &job.ids {
626                        if let Some(r) = outcome.results.get_mut(id) {
627                            r.outcome = TaskOutcome::Deferred;
628                        }
629                    }
630                }
631                Err(e) => log::warn!("pulse write task panicked: {e}"),
632            }
633        }
634        done
635    });
636
637    outcome.stats.retries = shared.retries.load(Ordering::SeqCst);
638    outcome.stats.calls_made = shared.calls.load(Ordering::SeqCst);
639    for (job, r) in finished {
640        match r {
641            Ok((text, usage)) => {
642                used_keys.push(job.key.clone());
643                for id in &job.ids {
644                    if let Some(res) = outcome.results.get_mut(id) {
645                        res.outcome = TaskOutcome::Called;
646                        res.text = Some(text.clone());
647                        res.usage = usage;
648                    }
649                }
650                if let Some(u) = usage {
651                    outcome.stats.input_tokens += u.input_tokens as u64;
652                    outcome.stats.output_tokens += u.output_tokens as u64;
653                    outcome.stats.cached_input_tokens += u.cached_input_tokens as u64;
654                }
655            }
656            Err(CallError::Fatal(msg)) | Err(CallError::Failed(msg)) => {
657                for id in &job.ids {
658                    if let Some(res) = outcome.results.get_mut(id) {
659                        res.outcome = TaskOutcome::Failed(msg.clone());
660                    }
661                }
662            }
663        }
664    }
665    if let Some(reason) = shared.stop.lock().unwrap().take() {
666        outcome.degraded = Some(reason);
667    }
668    verdict(outcome, used_keys, cache, cfg, start)
669}
670
671/// Call one job with retries; write the cache on success.
672async fn execute(
673    provider: Arc<dyn LlmProvider>,
674    job: &Job,
675    cfg: &WriteConfig,
676    shared: &Shared,
677    cache: &WriteCache,
678) -> CallResult {
679    let task = &job.task;
680    let started = Instant::now();
681    let mut max_tokens = task.max_tokens;
682    let mut grown = false;
683    let mut attempt = 0u32;
684    loop {
685        if shared.stopped() {
686            return Err(CallError::Failed("run stopped".into()));
687        }
688        attempt += 1;
689        let req = CompletionRequest {
690            system: &task.system,
691            user: &task.user,
692            output: task.output.as_mode(),
693            max_tokens,
694            temperature: 0.0,
695            tag: &task.id,
696        };
697        shared.calls.fetch_add(1, Ordering::SeqCst);
698        match provider.complete_request(&req).await {
699            Ok(resp) => {
700                if let Some(u) = resp.usage {
701                    shared.tokens_used.fetch_add(
702                        u.input_tokens as u64 + u.output_tokens as u64,
703                        Ordering::SeqCst,
704                    );
705                }
706                // Truncated JSON never parses, and a reasoning model may spend the whole
707                // budget before answering: retry once with twice the room.
708                if resp.stop == StopReason::MaxTokens
709                    && (task.output != OutputSpec::Text || resp.text.trim().is_empty())
710                    && !grown
711                {
712                    grown = true;
713                    max_tokens = max_tokens.saturating_mul(2);
714                    shared.retries.fetch_add(1, Ordering::SeqCst);
715                    continue;
716                }
717                if let StopReason::Other(reason) = &resp.stop
718                    && reason == "refusal"
719                {
720                    return Err(fail(shared, cfg, format!("{}: model refused", task.id)));
721                }
722                shared.consecutive_failures.store(0, Ordering::SeqCst);
723                let entry = CacheEntry {
724                    key: job.key.clone(),
725                    task_id: task.id.clone(),
726                    task_kind: task.kind.to_string(),
727                    model: resp.model.clone(),
728                    usage: resp.usage,
729                    text: resp.text.trim().to_string(),
730                };
731                if let Err(e) = cache.put(&entry) {
732                    log::warn!("pulse write cache: {e:#}");
733                }
734                eprintln!(
735                    "  Writing: {} (done, {:.1}s)",
736                    task.id,
737                    started.elapsed().as_secs_f64()
738                );
739                return Ok((entry.text, resp.usage));
740            }
741            Err(e) => {
742                let classified = e.downcast_ref::<ProviderError>();
743                let kind = classified.map(|p| p.kind);
744                if kind.is_some_and(|k| k.is_fatal()) {
745                    let reason = format!("{e}");
746                    shared.stop_with(reason.clone());
747                    return Err(CallError::Fatal(reason));
748                }
749                // Unclassified errors come from legacy `complete()` paths: retry them.
750                let retryable = kind.is_none_or(|k| k.is_retryable());
751                if retryable && attempt < cfg.max_attempts {
752                    if kind == Some(ProviderErrorKind::RateLimited) {
753                        shared.halve_concurrency();
754                    }
755                    let delay = classified
756                        .and_then(|p| p.retry_after)
757                        .unwrap_or_else(|| backoff(cfg.base_backoff, attempt, &task.id));
758                    log::debug!(
759                        "pulse write: {} attempt {attempt} failed: {e}; retry in {delay:?}",
760                        task.id
761                    );
762                    shared.retries.fetch_add(1, Ordering::SeqCst);
763                    tokio::time::sleep(delay).await;
764                    continue;
765                }
766                eprintln!(
767                    "  Writing: {} (failed, {:.1}s): {e}",
768                    task.id,
769                    started.elapsed().as_secs_f64()
770                );
771                return Err(fail(shared, cfg, format!("{e}")));
772            }
773        }
774    }
775}
776
777fn fail(shared: &Shared, cfg: &WriteConfig, msg: String) -> CallError {
778    let n = shared.consecutive_failures.fetch_add(1, Ordering::SeqCst) + 1;
779    if n >= cfg.breaker_threshold {
780        shared.stop_with(format!("{n} consecutive LLM failures; last: {msg}"));
781    }
782    CallError::Failed(msg)
783}
784
785/// `min(30s, base · 2^(attempt-1))` plus up to 25% jitter derived from the task id,
786/// so reruns wait the same way.
787fn backoff(base: Duration, attempt: u32, id: &str) -> Duration {
788    let exp = base.saturating_mul(1u32 << (attempt - 1).min(5));
789    let capped = exp.min(Duration::from_secs(30));
790    let h = blake3::hash(id.as_bytes()).as_bytes()[0] as u32;
791    capped + capped.mul_f64(h as f64 / 255.0 * 0.25)
792}
793
794fn verdict(
795    mut outcome: WriteOutcome,
796    used_keys: Vec<String>,
797    cache: &WriteCache,
798    cfg: &WriteConfig,
799    start: Instant,
800) -> WriteOutcome {
801    if outcome.dry_run {
802        return finish(outcome, start);
803    }
804    let mut ok = 0usize;
805    let mut eligible = 0usize;
806    for r in outcome.results.values() {
807        match r.outcome {
808            TaskOutcome::Cached | TaskOutcome::Called => {
809                ok += 1;
810                eligible += 1;
811            }
812            TaskOutcome::Failed(_) => eligible += 1,
813            _ => {}
814        }
815    }
816    if outcome.degraded.is_none()
817        && cfg.mode == LlmMode::On
818        && eligible > 0
819        && (ok as f32) / (eligible as f32) < cfg.degrade_threshold
820    {
821        outcome.degraded = Some(format!(
822            "only {ok} of {eligible} LLM sections succeeded (need {:.0}%)",
823            cfg.degrade_threshold * 100.0
824        ));
825    }
826    let complete = outcome.degraded.is_none() && !outcome.truncated;
827    if complete && cfg.prune && cfg.mode == LlmMode::On && eligible > 0 {
828        match cache.record_run_and_prune(used_keys, cfg.keep_runs) {
829            Ok(n) => outcome.stats.pruned = n,
830            Err(e) => log::warn!("pulse write cache prune failed: {e:#}"),
831        }
832    }
833    finish(outcome, start)
834}
835
836fn finish(mut outcome: WriteOutcome, start: Instant) -> WriteOutcome {
837    let s = &mut outcome.stats;
838    for r in outcome.results.values() {
839        match r.outcome {
840            TaskOutcome::Cached => s.cached += 1,
841            TaskOutcome::Called => s.called += 1,
842            TaskOutcome::Suppressed(_) => s.suppressed += 1,
843            TaskOutcome::Deferred => s.deferred += 1,
844            TaskOutcome::NotCached => s.not_cached += 1,
845            TaskOutcome::Failed(_) => s.failed += 1,
846            TaskOutcome::Skipped => s.skipped += 1,
847        }
848    }
849    if outcome.dry_run {
850        // Nothing was called; "Called" in the plan means "would call".
851        s.called = 0;
852    }
853    s.elapsed = start.elapsed();
854    outcome
855}