Skip to main content

aft/bash_background/
registry.rs

1use std::collections::{BTreeMap, HashMap, HashSet, VecDeque};
2#[cfg(not(windows))]
3use std::ffi::OsString;
4use std::fs;
5use std::io::Write;
6use std::path::{Path, PathBuf};
7#[cfg(windows)]
8use std::process::Command;
9use std::process::{Child, Stdio};
10use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
11#[cfg(unix)]
12use std::sync::OnceLock;
13use std::sync::{Arc, Mutex, RwLock};
14use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
15
16use crate::db::TrackedConnection;
17use serde::Serialize;
18
19use crate::bash_permissions::PermissionAsk;
20use crate::compress::caps::DropClass;
21#[cfg(unix)]
22use crate::compress::single_top_level_pipeline;
23use crate::compress::CompressionResult;
24use crate::context::SharedProgressSender;
25use crate::db::compression_events::CompressionAggregateCache;
26use crate::harness::Harness;
27use crate::list_envelope::ListEnvelope;
28use crate::protocol::{BashCompletedFrame, BashLongRunningFrame, BashPatternMatchFrame, PushFrame};
29use crate::sandbox_spawn::SpawnPlan;
30
31#[cfg(windows)]
32use std::os::windows::process::CommandExt;
33
34use super::buffer::{combine_streams, BgBuffer, DiskTruncation, StreamKind, TokenCountInput};
35use super::output::{
36    cap_completion_output, cap_completion_output_with_marker, cap_final_output,
37    cap_final_output_with_marker, completion_preview_threshold, json_output_pointer, quote_path,
38    retained_json_output_pointer, COMPRESS_INPUT_CAP_BYTES, COMPRESS_INPUT_HEAD_BYTES,
39    COMPRESS_INPUT_TAIL_BYTES, FINAL_OUTPUT_CAP_BYTES, RAW_PASSTHROUGH_CAP_BYTES,
40    RAW_PASSTHROUGH_HEAD_BYTES, RAW_PASSTHROUGH_TAIL_BYTES, RUNNING_OUTPUT_PREVIEW_BYTES,
41    STRUCTURED_OUTPUT_CAP_BYTES,
42};
43use super::persistence::{
44    allocate_task_layout, delete_resolved_task, delete_task_bundle, discover_task_ids,
45    open_task_artifact, quarantine_invalid_entry, quarantine_task_layout, read_exit_marker,
46    read_task_at, resolve_task_layout, session_tasks_dir, uninitialized_layout_is_recent,
47    unix_millis, update_task_at, validate_task_id, write_kill_marker_if_absent, write_task_at,
48    BgMode, ExitMarker, PersistedTask, TaskArtifact, TaskIoHandles, TaskPaths,
49};
50#[cfg(unix)]
51use super::process::terminate_pgid;
52#[cfg(windows)]
53use super::process::terminate_pid;
54use super::process::{
55    is_process_alive, is_recorded_process_alive, live_process_group_members, LiveDescendant,
56};
57use super::pty_process::spawn_pty_for_command;
58use super::pty_runtime::PtyRuntime;
59use super::watches::{
60    PatternMatch, WatchPattern, WatchRegistry, WATCH_TARGET_ERASED_CONTEXT,
61    WATCH_TARGET_ERASED_TEXT, WATCH_TASK_EXIT_TEXT,
62};
63use super::{BgTaskInfo, BgTaskStatus};
64use crate::db::bash_tasks::BashTaskRow;
65use crate::db::bash_watches::BashPatternWatchRow;
66/// Default timeout for background bash tasks: 30 minutes.
67/// Agents can override per-call via the `timeout` parameter (in ms).
68const DEFAULT_BG_TIMEOUT: Duration = Duration::from_secs(30 * 60);
69const PERSISTED_GC_GRACE: Duration = Duration::from_secs(24 * 60 * 60);
70const QUARANTINE_GC_GRACE: Duration = Duration::from_secs(30 * 24 * 60 * 60);
71
72const TOKENIZE_CAP_BYTES_PER_STREAM: usize = 128 * 1024;
73pub const ROOT_RECLAIMED_REASON: &str = "root_reclaimed";
74#[cfg(target_os = "linux")]
75pub(crate) const LINUX_SCOPE_ENV: &str = "AFT_INTERNAL_LINUX_SCOPE";
76
77#[derive(Debug, Clone, Serialize)]
78pub struct BgCompletion {
79    pub task_id: String,
80    /// Intentionally omitted from serialized completion payloads: push frames
81    /// carry `session_id` at the BashCompletedFrame envelope level for routing.
82    #[serde(skip_serializing)]
83    pub session_id: String,
84    pub status: BgTaskStatus,
85    pub exit_code: Option<i32>,
86    pub command: String,
87    /// Small head+tail preview of the cached terminal render at completion time,
88    /// cached so push-frame consumers and `bash_drain_completions` callers see
89    /// the same preview without racing against later output rotation. Empty
90    /// when not captured (e.g., persisted task seen on startup before buffer
91    /// reattachment).
92    #[serde(default, skip_serializing_if = "String::is_empty")]
93    pub output_preview: String,
94    #[serde(default, skip_serializing_if = "Option::is_none")]
95    pub bash_output_list_envelope: Option<ListEnvelope>,
96    /// True when the captured tail is shorter than the actual output (because
97    /// rotation occurred or the output exceeds the preview cap). Plugins use
98    /// this to render a `…` prefix and signal that `bash_status` would return
99    /// more.
100    #[serde(default, skip_serializing_if = "is_false")]
101    pub output_truncated: bool,
102    /// Token count for raw stdout+stderr before compression. Omitted when any
103    /// stream exceeds the 128 KiB tokenization cap.
104    #[serde(default, skip_serializing_if = "Option::is_none")]
105    pub original_tokens: Option<u32>,
106    /// Token count for the compressed output generated from the same capped
107    /// raw payload. Omitted when raw tokenization is skipped.
108    #[serde(default, skip_serializing_if = "Option::is_none")]
109    pub compressed_tokens: Option<u32>,
110    /// True when a stream exceeded the tokenization cap and counts are absent.
111    #[serde(default, skip_serializing_if = "is_false")]
112    pub tokens_skipped: bool,
113    #[serde(default, skip_serializing_if = "Option::is_none")]
114    pub status_reason: Option<String>,
115    pub live_descendants: Option<Vec<LiveDescendant>>,
116    #[serde(default, skip_serializing_if = "is_zero_usize")]
117    pub live_descendants_omitted: usize,
118    #[serde(default, skip_serializing_if = "Option::is_none")]
119    pub live_descendants_summary: Option<String>,
120}
121
122fn is_zero_usize(value: &usize) -> bool {
123    *value == 0
124}
125
126fn is_false(v: &bool) -> bool {
127    !*v
128}
129
130#[derive(Debug, Clone, Serialize)]
131pub struct BgTaskSnapshot {
132    #[serde(flatten)]
133    pub info: BgTaskInfo,
134    pub exit_code: Option<i32>,
135    pub child_pid: Option<u32>,
136    pub workdir: String,
137    pub output_preview: String,
138    #[serde(default, skip_serializing_if = "Option::is_none")]
139    pub bash_output_list_envelope: Option<ListEnvelope>,
140    pub output_truncated: bool,
141    pub output_path: Option<String>,
142    pub stderr_path: Option<String>,
143    #[serde(skip_serializing_if = "Option::is_none")]
144    pub pty_rows: Option<u16>,
145    #[serde(skip_serializing_if = "Option::is_none")]
146    pub pty_cols: Option<u16>,
147    #[serde(skip_serializing_if = "Option::is_none")]
148    pub pty_screen: Option<String>,
149    #[serde(default, skip_serializing_if = "Vec::is_empty")]
150    pub scanner_report: Vec<PermissionAsk>,
151    #[serde(default, skip_serializing_if = "is_false")]
152    pub sandbox_native: bool,
153    #[serde(default, skip_serializing_if = "is_false")]
154    pub sandbox_unavailable: bool,
155    pub live_descendants: Option<Vec<LiveDescendant>>,
156    #[serde(default, skip_serializing_if = "is_zero_usize")]
157    pub live_descendants_omitted: usize,
158    #[serde(default, skip_serializing_if = "Option::is_none")]
159    pub live_descendants_summary: Option<String>,
160    #[serde(default, skip_serializing_if = "is_false")]
161    pub kill_signaled: bool,
162    #[serde(default, skip_serializing_if = "is_zero_usize")]
163    pub kill_reached: usize,
164}
165
166#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
167pub struct BgTaskHealthCounts {
168    pub running: usize,
169    pub pending_completions: usize,
170}
171
172#[derive(Debug, Clone, Copy, PartialEq, Eq)]
173enum TerminalOutputKind {
174    Compressed,
175    Raw,
176    Structured,
177}
178
179#[derive(Debug, Clone, PartialEq, Eq)]
180struct TerminalOutputCache {
181    output_preview: String,
182    output_truncated: bool,
183    compression_input_line_count: Option<usize>,
184    kind: TerminalOutputKind,
185    output_path: Option<String>,
186    stderr_path: Option<String>,
187    artifact_access: ArtifactRecoveryAccess,
188    recovery: Option<RecoveryContext>,
189}
190
191#[derive(Debug, Clone, PartialEq, Eq)]
192struct ArtifactRecoveryAccess {
193    task_id: String,
194    readable: bool,
195}
196
197#[derive(Debug, Clone, PartialEq, Eq)]
198struct RecoveryContext {
199    dropped_by_class: BTreeMap<DropClass, usize>,
200    had_inner_drop: bool,
201    offset_hint_eligible: bool,
202    offset_start_line: Option<usize>,
203    byte_truncated: bool,
204    disk_truncated_prefix_bytes: u64,
205    output_path: Option<String>,
206    stderr_path: Option<String>,
207    include_stderr_path: bool,
208    artifact_access: ArtifactRecoveryAccess,
209}
210
211fn optional_string_bytes(value: Option<&String>) -> u64 {
212    value
213        .map(|value| crate::memory::usize_to_u64(value.len()))
214        .unwrap_or(0)
215}
216
217fn terminal_output_cache_estimated_bytes(cache: &TerminalOutputCache) -> u64 {
218    let recovery_bytes = cache
219        .recovery
220        .as_ref()
221        .map(|recovery| {
222            crate::memory::usize_to_u64(recovery.dropped_by_class.len())
223                .saturating_mul(
224                    (std::mem::size_of::<DropClass>() + std::mem::size_of::<usize>()) as u64,
225                )
226                .saturating_add(optional_string_bytes(recovery.output_path.as_ref()))
227                .saturating_add(optional_string_bytes(recovery.stderr_path.as_ref()))
228                .saturating_add(crate::memory::usize_to_u64(
229                    recovery.artifact_access.task_id.len(),
230                ))
231        })
232        .unwrap_or(0);
233    (std::mem::size_of::<TerminalOutputCache>() as u64)
234        .saturating_add(crate::memory::usize_to_u64(cache.output_preview.len()))
235        .saturating_add(optional_string_bytes(cache.output_path.as_ref()))
236        .saturating_add(optional_string_bytes(cache.stderr_path.as_ref()))
237        .saturating_add(crate::memory::usize_to_u64(
238            cache.artifact_access.task_id.len(),
239        ))
240        .saturating_add(recovery_bytes)
241}
242
243fn completion_estimated_bytes(completion: &BgCompletion) -> u64 {
244    (std::mem::size_of::<BgCompletion>() as u64)
245        .saturating_add(crate::memory::usize_to_u64(completion.task_id.len()))
246        .saturating_add(crate::memory::usize_to_u64(completion.session_id.len()))
247        .saturating_add(crate::memory::usize_to_u64(completion.command.len()))
248        .saturating_add(crate::memory::usize_to_u64(completion.output_preview.len()))
249}
250
251impl RecoveryContext {
252    fn has_visible_drop(&self) -> bool {
253        self.byte_truncated
254            || self.disk_truncated_prefix_bytes > 0
255            || self.had_inner_drop
256            || !self.dropped_by_class.is_empty()
257    }
258}
259
260/// How much replay recovery a session task directory has seen.
261///
262/// `in_flight` answers "is a recovery running right now"; `finished` makes a
263/// recovery that started and ended inside someone else's observation window
264/// visible, which a flag alone cannot do. Comparing a snapshot against the
265/// current value therefore answers "did any recovery of this session touch it
266/// since I looked", which is what a destructive sweep needs to know.
267#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
268struct SessionRecovery {
269    in_flight: usize,
270    finished: u64,
271}
272
273/// Marks a session's task directory as under replay recovery for as long as it
274/// lives. Replay holds one per session it restores; the persisted GC skips a
275/// session whose recovery is in flight and abandons any action on a session a
276/// recovery touched while it was deciding.
277#[doc(hidden)]
278pub struct SessionRecoveryGuard {
279    registry: BgTaskRegistry,
280    key: Option<String>,
281}
282
283impl Drop for SessionRecoveryGuard {
284    fn drop(&mut self) {
285        let Some(key) = self.key.take() else {
286            return;
287        };
288        let mut recoveries = self
289            .registry
290            .inner
291            .session_recovery
292            .lock()
293            .unwrap_or_else(std::sync::PoisonError::into_inner);
294        let Some(recovery) = recoveries.get_mut(&key) else {
295            return;
296        };
297        recovery.in_flight = recovery.in_flight.saturating_sub(1);
298        recovery.finished = recovery.finished.wrapping_add(1);
299        // The entry outlives the recovery on purpose: a sweep that snapshotted
300        // this session before the recovery started must still be able to see
301        // that one ran, and a removed entry would read as "never touched".
302    }
303}
304
305#[derive(Clone)]
306pub struct BgTaskRegistry {
307    pub(crate) inner: Arc<RegistryInner>,
308}
309
310/// What woke the watchdog pass in which a task was observed terminal.
311#[derive(Debug, Clone, Copy, PartialEq, Eq)]
312pub enum WatchdogPassCause {
313    /// A `wake_tx` send (reader or waiter signalled).
314    Wake,
315    /// The periodic ticker.
316    Tick,
317}
318
319pub(crate) struct RegistryInner {
320    pub(crate) tasks: Mutex<HashMap<String, Arc<BgTask>>>,
321    pub(crate) completions: Mutex<VecDeque<BgCompletion>>,
322    pub(crate) progress_sender: SharedProgressSender,
323    watchdog_started: AtomicBool,
324    pub(crate) shutdown: AtomicBool,
325    pub(crate) long_running_reminder_enabled: AtomicBool,
326    pub(crate) long_running_reminder_interval_ms: AtomicU64,
327    persisted_gc_started: AtomicBool,
328    #[cfg(test)]
329    persisted_gc_runs: AtomicU64,
330    /// Name of the thread the once-per-process persisted GC ran on. Lets the
331    /// integration suite pin that the GC stays off the configure/replay thread
332    /// (the replay caller is the standalone request loop).
333    persisted_gc_thread: Mutex<Option<String>>,
334    /// Replay-recovery activity per session task directory, keyed by the
335    /// directory's name. Written by `SessionRecoveryGuard`, read by the
336    /// persisted GC so a sweep never judges a session that replay is in the
337    /// middle of restoring.
338    session_recovery: Mutex<HashMap<String, SessionRecovery>>,
339    /// Output compression callback. Set by `AppContext` after construction.
340    /// Takes (command, raw_output, exit_code) and returns compressed text. Called from
341    /// the watchdog thread when a task reaches a terminal state and from
342    /// `bash_status`/`list` snapshot reads. When `None`, output is returned
343    /// uncompressed.
344    pub(crate) compressor:
345        Mutex<Option<Box<dyn Fn(&str, String, Option<i32>) -> CompressionResult + Send + Sync>>>,
346    pub(crate) db_pool: RwLock<Option<Arc<Mutex<TrackedConnection>>>>,
347    pub(crate) db_harness: RwLock<Option<String>>,
348    pub(crate) compression_aggregates: Arc<CompressionAggregateCache>,
349    pub(crate) wake_tx: crossbeam_channel::Sender<()>,
350    pub(crate) wake_rx: crossbeam_channel::Receiver<()>,
351    terminal_transition: tokio::sync::Notify,
352    /// Which watchdog pass observed each task reach a terminal state: the
353    /// wake channel or the periodic ticker. Tests assert the mechanism from
354    /// this record instead of racing the 500 ms ticker on the wall clock.
355    pub(crate) completion_pass_cause: Mutex<HashMap<String, WatchdogPassCause>>,
356    pub(crate) watch_registry: Mutex<WatchRegistry>,
357    /// Session identities with an installed route for this root in subc mode.
358    /// The loop-owned route table refreshes this snapshot before replay and on
359    /// bind/unbind transitions; standalone replay relies on its binding-session
360    /// exception instead.
361    live_delivery_sessions: Mutex<HashSet<String>>,
362    wait_detach_sessions: Mutex<HashSet<String>>,
363    active_wait_sessions: Mutex<HashMap<String, usize>>,
364    wait_registered_tasks: Mutex<HashMap<String, HashSet<String>>>,
365}
366
367pub(crate) struct BgTask {
368    pub(crate) task_id: String,
369    pub(crate) session_id: String,
370    pub(crate) paths: TaskPaths,
371    artifact_root: PathBuf,
372    pub(crate) started: Instant,
373    pub(crate) last_reminder_at: Mutex<Option<Instant>>,
374    pub(crate) terminal_at: Mutex<Option<Instant>>,
375    pub(crate) state: Mutex<BgTaskState>,
376}
377
378pub(crate) enum TaskRuntime {
379    Piped(Option<Child>),
380    Pty(Option<PtyRuntime>),
381}
382
383pub(crate) struct BgTaskState {
384    pub(crate) metadata: PersistedTask,
385    pub(crate) runtime: TaskRuntime,
386    /// Pinned task-io directory and original O_EXCL output handles retained for
387    /// the process lifetime. Daemon writes never reopen child-writable names.
388    pub(crate) io_handles: Option<TaskIoHandles>,
389    pub(crate) detached: bool,
390    /// True once `reap_child` has observed the direct child handle's exit
391    /// via `try_wait()`. Used by the two-pass watchdog to skip the racy
392    /// `is_process_alive(child_pid)` probe on the second pass — we already
393    /// have authoritative evidence that the child is dead, no need to
394    /// re-verify via PID liveness which is unreliable on Windows where
395    /// PIDs can be recycled within seconds.
396    ///
397    /// Remains `false` on replay-restored tasks (those have a `child_pid`
398    /// but never observed exit via this process's `try_wait()`), so those
399    /// continue to fall through to the `is_process_alive` probe path.
400    pub(crate) child_exit_observed: bool,
401    /// Prevent duplicate completion delivery while the post-exit process-group
402    /// sample is waiting for its grace period.
403    pub(crate) descendant_sampling_started: bool,
404    pub(crate) buffer: BgBuffer,
405    terminal_output_cache: Option<TerminalOutputCache>,
406    /// PTY-only: set for timeout kill intent before signaling the child.
407    pub(crate) pending_terminal_override: Option<BgTaskStatus>,
408}
409
410fn completion_matches_session(completion: &BgCompletion, session_id: Option<&str>) -> bool {
411    session_id
412        .map(|session_id| completion.session_id == session_id)
413        .unwrap_or(true)
414}
415
416impl BgTaskRegistry {
417    /// The watchdog pass cause recorded when `task_id` was first observed
418    /// terminal, if the watchdog (rather than a status poll) observed it.
419    pub fn completion_pass_cause(&self, task_id: &str) -> Option<WatchdogPassCause> {
420        self.inner
421            .completion_pass_cause
422            .lock()
423            .ok()
424            .and_then(|map| map.get(task_id).copied())
425    }
426
427    pub fn new(progress_sender: SharedProgressSender) -> Self {
428        let (wake_tx, wake_rx) = crossbeam_channel::bounded(1);
429        Self {
430            inner: Arc::new(RegistryInner {
431                tasks: Mutex::new(HashMap::new()),
432                completions: Mutex::new(VecDeque::new()),
433                progress_sender,
434                watchdog_started: AtomicBool::new(false),
435                shutdown: AtomicBool::new(false),
436                long_running_reminder_enabled: AtomicBool::new(true),
437                long_running_reminder_interval_ms: AtomicU64::new(600_000),
438                persisted_gc_started: AtomicBool::new(false),
439                #[cfg(test)]
440                persisted_gc_runs: AtomicU64::new(0),
441                persisted_gc_thread: Mutex::new(None),
442                session_recovery: Mutex::new(HashMap::new()),
443                compressor: Mutex::new(None),
444                db_pool: RwLock::new(None),
445                db_harness: RwLock::new(None),
446                compression_aggregates: Arc::new(CompressionAggregateCache::default()),
447                wake_tx,
448                wake_rx,
449                terminal_transition: tokio::sync::Notify::new(),
450                completion_pass_cause: Mutex::new(HashMap::new()),
451                watch_registry: Mutex::new(WatchRegistry::default()),
452                live_delivery_sessions: Mutex::new(HashSet::new()),
453                wait_detach_sessions: Mutex::new(HashSet::new()),
454                active_wait_sessions: Mutex::new(HashMap::new()),
455                wait_registered_tasks: Mutex::new(HashMap::new()),
456            }),
457        }
458    }
459
460    /// Return whether `path` is an exact artifact registered to `session_id`.
461    ///
462    /// The requested path is canonicalized and compared with exact artifact
463    /// names under the task directory identity captured at registration time.
464    /// Deliberately do not grant access by `bash-tasks` directory prefix: a
465    /// prefix exception would expose unrelated files and could be widened
466    /// through symlinks or directory replacement.
467    pub fn is_session_owned_artifact_path(&self, session_id: &str, path: &Path) -> bool {
468        let Ok(requested) = fs::canonicalize(path) else {
469            return false;
470        };
471        let Ok(tasks) = self.inner.tasks.lock() else {
472            return false;
473        };
474
475        tasks.values().any(|task| {
476            task.session_id == session_id
477                && [
478                    &task.paths.stdout,
479                    &task.paths.stderr,
480                    &task.paths.exit,
481                    &task.paths.pty,
482                ]
483                .into_iter()
484                .filter_map(|known| known.file_name())
485                .any(|name| task.artifact_root.join(name) == requested)
486        })
487    }
488
489    pub fn read_artifact_path(
490        &self,
491        session_id: &str,
492        path: &Path,
493    ) -> Option<Result<Vec<u8>, String>> {
494        let requested = fs::canonicalize(path).ok()?;
495        let tasks = self.inner.tasks.lock().ok()?;
496        let (task, artifact) = tasks.values().find_map(|task| {
497            if task.session_id != session_id {
498                return None;
499            }
500            TaskArtifact::ALL.into_iter().find_map(|artifact| {
501                let expected = task
502                    .paths
503                    .artifact_path(artifact)
504                    .file_name()
505                    .map(|name| task.artifact_root.join(name));
506                (expected.as_deref() == Some(requested.as_path()))
507                    .then(|| (Arc::clone(task), artifact))
508            })
509        })?;
510        drop(tasks);
511        Some(self.read_artifact(&task.task_id, session_id, artifact))
512    }
513
514    pub fn read_artifact_range(
515        &self,
516        task_id: &str,
517        session_id: &str,
518        artifact: TaskArtifact,
519        offset: u64,
520    ) -> Result<(Vec<u8>, u64), String> {
521        validate_task_id(task_id).map_err(|error| error.to_string())?;
522        let task = self
523            .task_for_session(task_id, session_id)
524            .ok_or_else(|| "task_not_found".to_string())?;
525        let mut file = open_task_artifact(&task.paths, artifact)
526            .map_err(|error| format!("artifact_refused: {error}"))?;
527        let len = file
528            .len()
529            .map_err(|error| format!("artifact_refused: {error}"))?;
530        let start = offset.min(len);
531        let bytes = file
532            .read_range(start, len.saturating_sub(start))
533            .map_err(|error| format!("artifact_refused: {error}"))?;
534        Ok((bytes, len))
535    }
536
537    pub fn read_artifact(
538        &self,
539        task_id: &str,
540        session_id: &str,
541        artifact: TaskArtifact,
542    ) -> Result<Vec<u8>, String> {
543        validate_task_id(task_id).map_err(|error| error.to_string())?;
544        let task = self
545            .task_for_session(task_id, session_id)
546            .ok_or_else(|| "task_not_found".to_string())?;
547        let mut file = open_task_artifact(&task.paths, artifact)
548            .map_err(|error| format!("artifact_refused: {error}"))?;
549        file.read_all()
550            .map_err(|error| format!("artifact_refused: {error}"))
551    }
552
553    pub fn set_harness(&self, harness: Harness) {
554        if let Ok(mut slot) = self.inner.db_harness.write() {
555            *slot = Some(harness.storage_segment());
556        }
557    }
558
559    pub fn set_db_pool(&self, conn: Arc<Mutex<TrackedConnection>>) {
560        if let Ok(mut slot) = self.inner.db_pool.write() {
561            *slot = Some(conn);
562        }
563        self.inner.compression_aggregates.clear();
564    }
565
566    pub fn clear_db_pool(&self) {
567        if let Ok(mut slot) = self.inner.db_pool.write() {
568            *slot = None;
569        }
570        self.inner.compression_aggregates.clear();
571    }
572
573    pub(crate) fn record_live_delivery_session(&self, session_id: &str) {
574        if let Ok(mut live_sessions) = self.inner.live_delivery_sessions.lock() {
575            live_sessions.insert(session_id.to_string());
576        }
577    }
578
579    pub(crate) fn replace_live_delivery_sessions(&self, sessions: HashSet<String>) {
580        if let Ok(mut live_sessions) = self.inner.live_delivery_sessions.lock() {
581            *live_sessions = sessions;
582        }
583    }
584
585    fn originating_session_has_live_route(&self, session_id: &str) -> bool {
586        self.inner
587            .live_delivery_sessions
588            .lock()
589            .map(|sessions| sessions.contains(session_id))
590            .unwrap_or(true)
591    }
592
593    fn should_retire_foreign_delivery(
594        &self,
595        originating_session_id: &str,
596        binding_session_id: &str,
597    ) -> bool {
598        originating_session_id != binding_session_id
599            && !self.originating_session_has_live_route(originating_session_id)
600    }
601
602    pub(crate) fn compression_aggregate_cache(&self) -> Arc<CompressionAggregateCache> {
603        Arc::clone(&self.inner.compression_aggregates)
604    }
605
606    pub fn register_foreground_task(&self, session_id: &str, task_id: &str) {
607        if let Ok(mut tasks) = self.inner.wait_registered_tasks.lock() {
608            tasks
609                .entry(session_id.to_string())
610                .or_default()
611                .insert(task_id.to_string());
612        }
613    }
614
615    pub fn begin_wait_mode_session(&self, session_id: &str, task_id: &str) {
616        if let Ok(mut active) = self.inner.active_wait_sessions.lock() {
617            *active.entry(session_id.to_string()).or_insert(0) += 1;
618        }
619        self.register_foreground_task(session_id, task_id);
620        if let Ok(mut detach) = self.inner.wait_detach_sessions.lock() {
621            detach.remove(session_id);
622        }
623    }
624
625    pub fn unregister_foreground_task(&self, session_id: &str, task_id: &str) {
626        if let Ok(mut tasks) = self.inner.wait_registered_tasks.lock() {
627            if let Some(session_tasks) = tasks.get_mut(session_id) {
628                session_tasks.remove(task_id);
629                if session_tasks.is_empty() {
630                    tasks.remove(session_id);
631                }
632            }
633        }
634    }
635
636    pub fn end_wait_mode_session(&self, session_id: &str, task_id: &str) {
637        let no_active_wait = if let Ok(mut active) = self.inner.active_wait_sessions.lock() {
638            match active.get_mut(session_id) {
639                Some(count) if *count > 1 => *count -= 1,
640                Some(_) => {
641                    active.remove(session_id);
642                }
643                None => {}
644            }
645            !active.contains_key(session_id)
646        } else {
647            false
648        };
649        self.unregister_foreground_task(session_id, task_id);
650        if no_active_wait {
651            if let Ok(mut detach) = self.inner.wait_detach_sessions.lock() {
652                detach.remove(session_id);
653            }
654        }
655    }
656
657    /// Kill the foreground bash task(s) still registered for an in-flight
658    /// call. Explicit background and PTY tasks are never registered
659    /// here, so an abort cannot affect those deliberately detached tasks.
660    pub fn abort_inflight(&self, session_id: &str) -> Result<usize, String> {
661        let task_ids = self
662            .inner
663            .wait_registered_tasks
664            .lock()
665            .map(|mut tasks| tasks.remove(session_id).unwrap_or_default())
666            .map_err(|_| "wait registration lock poisoned".to_string())?;
667        if let Ok(mut active) = self.inner.active_wait_sessions.lock() {
668            active.remove(session_id);
669        }
670        if let Ok(mut detach) = self.inner.wait_detach_sessions.lock() {
671            detach.remove(session_id);
672        }
673
674        let mut killed = 0;
675        for task_id in task_ids {
676            let Some(task) = self.task_for_session(&task_id, session_id) else {
677                continue;
678            };
679            let is_terminal = task
680                .state
681                .lock()
682                .map(|state| state.metadata.status.is_terminal())
683                .map_err(|_| "background task lock poisoned".to_string())?;
684            if is_terminal {
685                continue;
686            }
687            let snapshot = self.kill_with_status_reason(
688                &task_id,
689                session_id,
690                BgTaskStatus::Killed,
691                Some("call_aborted".to_string()),
692            )?;
693            if snapshot.info.status == BgTaskStatus::Killed
694                && snapshot.info.status_reason.as_deref() == Some("call_aborted")
695            {
696                killed += 1;
697            }
698        }
699        Ok(killed)
700    }
701
702    pub fn signal_wait_mode_detach(&self, session_id: &str) -> bool {
703        let is_waiting = self
704            .inner
705            .active_wait_sessions
706            .lock()
707            .map(|active| active.get(session_id).copied().unwrap_or(0) > 0)
708            .unwrap_or(false);
709        if !is_waiting {
710            return false;
711        }
712        self.inner
713            .wait_detach_sessions
714            .lock()
715            .map(|mut detach| detach.insert(session_id.to_string()))
716            .unwrap_or(false)
717    }
718
719    /// Number of sessions currently blocked in a `wait: true` foreground bash.
720    /// Diagnostic only (the detach-signal trace log).
721    pub fn active_wait_session_count(&self) -> usize {
722        self.inner
723            .active_wait_sessions
724            .lock()
725            .map(|active| active.len())
726            .unwrap_or(0)
727    }
728
729    pub fn take_wait_mode_detach(&self, session_id: &str) -> bool {
730        self.inner
731            .wait_detach_sessions
732            .lock()
733            .map(|mut detach| detach.remove(session_id))
734            .unwrap_or(false)
735    }
736
737    pub(crate) fn wait_mode_detach_pending(&self, session_id: &str) -> bool {
738        self.inner
739            .wait_detach_sessions
740            .lock()
741            .map(|detach| detach.contains(session_id))
742            .unwrap_or(false)
743    }
744
745    pub(crate) async fn terminal_transition_notified(&self) {
746        self.inner.terminal_transition.notified().await;
747    }
748
749    /// Install the output-compression callback. Called by `main.rs` after
750    /// `AppContext` is constructed so that snapshot/completion paths can
751    /// invoke `compress::compress_with_registry` without holding a context
752    /// reference. When called multiple times, the latest installation wins.
753    pub fn set_compressor<F>(&self, compressor: F)
754    where
755        F: Fn(&str, String) -> CompressionResult + Send + Sync + 'static,
756    {
757        self.set_compressor_with_exit_code(move |command, output, _exit_code| {
758            compressor(command, output)
759        });
760    }
761
762    pub fn set_compressor_with_exit_code<F>(&self, compressor: F)
763    where
764        F: Fn(&str, String, Option<i32>) -> CompressionResult + Send + Sync + 'static,
765    {
766        if let Ok(mut slot) = self.inner.compressor.lock() {
767            *slot = Some(Box::new(compressor));
768        }
769    }
770
771    /// Apply the installed compressor (if any) to `output`. Returns `output`
772    /// untouched when no compressor is installed.
773    pub(crate) fn compress_output(
774        &self,
775        command: &str,
776        output: String,
777        exit_code: Option<i32>,
778    ) -> CompressionResult {
779        let Ok(slot) = self.inner.compressor.lock() else {
780            return CompressionResult::new(output);
781        };
782        match slot.as_ref() {
783            Some(compressor) => compressor(command, output, exit_code),
784            None => CompressionResult::new(output),
785        }
786    }
787
788    fn ensure_terminal_output_cache(&self, task: &Arc<BgTask>) -> Option<TerminalOutputCache> {
789        let (metadata, buffer) = {
790            let state = task.state.lock().ok()?;
791            if !state.metadata.status.is_terminal() || state.metadata.mode == BgMode::Pty {
792                return None;
793            }
794            if let Some(cache) = state.terminal_output_cache.clone() {
795                return Some(cache);
796            }
797            (state.metadata.clone(), state.buffer.clone())
798        };
799
800        let mut cap_buffer = buffer.clone();
801        let disk_truncation = cap_buffer.enforce_terminal_cap();
802        let cache =
803            self.render_terminal_output(&metadata, &cap_buffer, disk_truncation, Some(&task.paths));
804        let mut state = task.state.lock().ok()?;
805        if !state.metadata.status.is_terminal() || state.metadata.mode == BgMode::Pty {
806            return None;
807        }
808        if let Some(existing) = state.terminal_output_cache.clone() {
809            return Some(existing);
810        }
811        state.terminal_output_cache = Some(cache.clone());
812        Some(cache)
813    }
814
815    fn render_terminal_output(
816        &self,
817        metadata: &PersistedTask,
818        buffer: &BgBuffer,
819        disk_truncation: DiskTruncation,
820        paths: Option<&TaskPaths>,
821    ) -> TerminalOutputCache {
822        let output_readable = buffer
823            .output_path()
824            .is_some_and(|path| self.is_session_owned_artifact_path(&metadata.session_id, &path));
825        let stderr_readable = buffer
826            .stderr_path()
827            .map(|path| self.is_session_owned_artifact_path(&metadata.session_id, path))
828            .unwrap_or(true);
829        let artifact_access = ArtifactRecoveryAccess {
830            task_id: metadata.task_id.clone(),
831            readable: output_readable && stderr_readable,
832        };
833
834        if metadata.mode == BgMode::Pty {
835            return TerminalOutputCache {
836                output_preview: String::new(),
837                output_truncated: false,
838                compression_input_line_count: None,
839                kind: TerminalOutputKind::Raw,
840                output_path: buffer.output_path().map(|path| path.display().to_string()),
841                stderr_path: buffer.stderr_path().map(|path| path.display().to_string()),
842                artifact_access,
843                recovery: None,
844            };
845        }
846
847        let mut rendered = if let Some(structured) = render_structured_output(
848            &metadata.command,
849            buffer,
850            disk_truncation,
851            artifact_access.clone(),
852        ) {
853            structured
854        } else if !metadata.compressed {
855            render_raw_passthrough(buffer, disk_truncation, artifact_access)
856        } else {
857            let raw = buffer.read_combined_head_tail(
858                COMPRESS_INPUT_CAP_BYTES,
859                COMPRESS_INPUT_HEAD_BYTES,
860                COMPRESS_INPUT_TAIL_BYTES,
861            );
862            let compressed = self.compress_output(&metadata.command, raw.text, metadata.exit_code);
863            render_compressed_with_recovery(
864                buffer,
865                compressed,
866                raw.truncated,
867                disk_truncation,
868                artifact_access,
869            )
870        };
871        normalize_piped_display_output(&mut rendered.output_preview);
872        append_pipeline_warning(&mut rendered, metadata, paths);
873        rendered
874    }
875
876    fn snapshot_with_terminal_cache(
877        &self,
878        task: &Arc<BgTask>,
879        preview_bytes: usize,
880    ) -> BgTaskSnapshot {
881        let mut snapshot = task.snapshot(preview_bytes);
882        self.maybe_compress_snapshot(task, &mut snapshot);
883        snapshot
884    }
885
886    fn post_terminal_transition(&self, task: &Arc<BgTask>, emit_frame: bool) -> Result<(), String> {
887        let should_sample = {
888            let mut state = task
889                .state
890                .lock()
891                .map_err(|_| "background task lock poisoned".to_string())?;
892            if !state.metadata.status.is_terminal() {
893                return Ok(());
894            }
895            if state.metadata.mode != BgMode::Pipes || state.descendant_sampling_started {
896                false
897            } else {
898                state.descendant_sampling_started = true;
899                true
900            }
901        };
902
903        // The wrapper shell waits for every pipeline member before it exits, so
904        // at the terminal transition the only survivors in the task's process
905        // group are reparented grandchildren (pool workers a SIGPIPE'd producer
906        // never reaped). Sample once, immediately: the common case is an empty
907        // group, and that case must not pay any delay because every foreground
908        // `bash` reply and `wait:true` result passes through here.
909        let survivors_at_exit = should_sample
910            && self
911                .sample_task_process_group(task)
912                .is_some_and(|(members, omitted)| !members.is_empty() || omitted > 0);
913        if !survivors_at_exit {
914            let result = self.finish_terminal_transition(task, emit_frame);
915            self.inner.terminal_transition.notify_waiters();
916            return result;
917        }
918
919        // Survivors exist. Give them a short grace before the frame names them
920        // (a grandchild that exits on its own within 300 ms is not the class
921        // being reported), then a longer settle for the row bash_status shows.
922        let registry = self.clone();
923        let task = Arc::clone(task);
924        std::thread::Builder::new()
925            .name(format!("aft-bg-descendants-{}", task.task_id))
926            .spawn(move || {
927                std::thread::sleep(Duration::from_millis(300));
928                let settled = registry.sample_task_process_group(&task);
929                if let Err(error) = registry.finish_terminal_transition(&task, emit_frame) {
930                    crate::slog_warn!(
931                        "failed to finish background task {} after descendant sampling: {error}",
932                        task.task_id
933                    );
934                }
935                registry.inner.terminal_transition.notify_waiters();
936                if settled
937                    .as_ref()
938                    .is_some_and(|(members, omitted)| !members.is_empty() || *omitted > 0)
939                {
940                    std::thread::sleep(Duration::from_millis(1_700));
941                    let _ = registry.sample_task_process_group(&task);
942                }
943            })
944            .map_err(|error| format!("failed to start descendant sampler: {error}"))?;
945        Ok(())
946    }
947
948    fn sample_task_process_group(
949        &self,
950        task: &Arc<BgTask>,
951    ) -> Option<(Vec<LiveDescendant>, usize)> {
952        let pgid = task
953            .state
954            .lock()
955            .ok()
956            .and_then(|state| state.metadata.pgid)?;
957        let sample = live_process_group_members(pgid);
958        if let Ok(mut state) = task.state.lock() {
959            state.metadata.live_descendants = sample.as_ref().map(|(members, _)| members.clone());
960            state.metadata.live_descendants_omitted =
961                sample.as_ref().map(|(_, omitted)| *omitted).unwrap_or(0);
962            if let Err(error) = self.persist_task(&task.paths, &state.metadata) {
963                crate::slog_warn!(
964                    "failed to persist live descendants for {}: {error}",
965                    task.task_id
966                );
967            }
968        }
969        sample
970    }
971
972    fn finish_terminal_transition(
973        &self,
974        task: &Arc<BgTask>,
975        emit_frame: bool,
976    ) -> Result<(), String> {
977        let (metadata, buffer) = {
978            let state = task
979                .state
980                .lock()
981                .map_err(|_| "background task lock poisoned".to_string())?;
982            if !state.metadata.status.is_terminal() {
983                return Ok(());
984            }
985            (state.metadata.clone(), state.buffer.clone())
986        };
987
988        let cache = self.ensure_terminal_output_cache(task);
989        self.enqueue_completion_from_parts(
990            &metadata,
991            Some(&buffer),
992            None,
993            emit_frame,
994            cache.as_ref(),
995        );
996        Ok(())
997    }
998
999    fn persist_task(&self, paths: &TaskPaths, metadata: &PersistedTask) -> std::io::Result<()> {
1000        let task = resolve_task_layout(&paths.session_dir, &paths.task_id)?;
1001        write_task_at(&task, metadata)?;
1002        self.dual_write_task(paths, metadata);
1003        Ok(())
1004    }
1005
1006    fn update_task_metadata<F>(
1007        &self,
1008        paths: &TaskPaths,
1009        update: F,
1010    ) -> std::io::Result<PersistedTask>
1011    where
1012        F: FnOnce(&mut PersistedTask),
1013    {
1014        let task = resolve_task_layout(&paths.session_dir, &paths.task_id)?;
1015        let metadata = update_task_at(&task, update)?;
1016        self.dual_write_task(paths, &metadata);
1017        Ok(metadata)
1018    }
1019
1020    fn dual_write_task(&self, paths: &TaskPaths, metadata: &PersistedTask) {
1021        let pool = self.inner.db_pool.read().ok().and_then(|slot| slot.clone());
1022        let Some(pool) = pool else {
1023            return;
1024        };
1025        let harness = self
1026            .inner
1027            .db_harness
1028            .read()
1029            .ok()
1030            .and_then(|slot| slot.clone());
1031        let Some(harness) = harness else {
1032            crate::slog_warn!(
1033                "dual-write bash_task to DB skipped for {}: harness not configured",
1034                metadata.task_id
1035            );
1036            return;
1037        };
1038        let row = match metadata.to_bash_task_row(&harness, paths) {
1039            Ok(row) => row,
1040            Err(error) => {
1041                crate::slog_warn!(
1042                    "dual-write bash_task to DB failed for {}: {}",
1043                    metadata.task_id,
1044                    error
1045                );
1046                return;
1047            }
1048        };
1049        let conn = match pool.lock() {
1050            Ok(conn) => conn,
1051            Err(_) => {
1052                crate::slog_warn!(
1053                    "dual-write bash_task to DB failed for {}: db mutex poisoned",
1054                    metadata.task_id
1055                );
1056                return;
1057            }
1058        };
1059        if let Err(error) = crate::db::bash_tasks::upsert_bash_task(&conn, &row) {
1060            crate::slog_warn!(
1061                "dual-write bash_task to DB failed for {}: {}",
1062                metadata.task_id,
1063                error
1064            );
1065        }
1066    }
1067
1068    fn delete_gc_task_from_db(&self, metadata: &PersistedTask) {
1069        let pool = self.inner.db_pool.read().ok().and_then(|slot| slot.clone());
1070        let Some(pool) = pool else {
1071            return;
1072        };
1073        let harness = self
1074            .inner
1075            .db_harness
1076            .read()
1077            .ok()
1078            .and_then(|slot| slot.clone());
1079        let Some(harness) = harness else {
1080            crate::slog_warn!(
1081                "GC bash_task DB delete skipped for {}: harness not configured",
1082                metadata.task_id
1083            );
1084            return;
1085        };
1086        let conn = match pool.lock() {
1087            Ok(conn) => conn,
1088            Err(_) => {
1089                crate::slog_warn!(
1090                    "GC bash_task DB delete failed for {}: db mutex poisoned",
1091                    metadata.task_id
1092                );
1093                return;
1094            }
1095        };
1096        if let Err(error) = crate::db::bash_tasks::delete_delivered_terminal_bash_task(
1097            &conn,
1098            &harness,
1099            &metadata.session_id,
1100            &metadata.task_id,
1101            "persisted_gc_delivered_terminal",
1102        ) {
1103            crate::slog_warn!(
1104                "GC bash_task DB delete failed for {}: {}",
1105                metadata.task_id,
1106                error
1107            );
1108        }
1109    }
1110
1111    fn persisted_task_process_is_alive(metadata: &PersistedTask) -> bool {
1112        let child_pid = metadata.child_pid;
1113        let group_leader = metadata.pgid.and_then(|pid| u32::try_from(pid).ok());
1114        child_pid
1115            .into_iter()
1116            .chain(group_leader)
1117            .any(|pid| is_recorded_process_alive(pid, metadata.started_at))
1118    }
1119
1120    fn db_has_live_process_for_task(&self, task_id: &str) -> bool {
1121        let Some((harness, pool)) = self.db_harness_and_pool() else {
1122            return false;
1123        };
1124        let Ok(conn) = pool.lock() else {
1125            return false;
1126        };
1127        crate::db::bash_tasks::list_bash_tasks_by_id(&conn, &harness, task_id)
1128            .map(|rows| {
1129                rows.into_iter().any(|row| {
1130                    let started_at = u64::try_from(row.started_at).unwrap_or_default();
1131                    row.pid
1132                        .and_then(|pid| u32::try_from(pid).ok())
1133                        .into_iter()
1134                        .chain(row.pgid.and_then(|pid| u32::try_from(pid).ok()))
1135                        .any(|pid| is_recorded_process_alive(pid, started_at))
1136                })
1137            })
1138            .unwrap_or(false)
1139    }
1140
1141    fn db_harness_and_pool(&self) -> Option<(String, Arc<Mutex<TrackedConnection>>)> {
1142        let pool = self
1143            .inner
1144            .db_pool
1145            .read()
1146            .ok()
1147            .and_then(|slot| slot.clone())?;
1148        let harness = self
1149            .inner
1150            .db_harness
1151            .read()
1152            .ok()
1153            .and_then(|slot| slot.clone())?;
1154        Some((harness, pool))
1155    }
1156
1157    pub fn pending_pattern_matches_for_session(
1158        &self,
1159        session_id: &str,
1160    ) -> Vec<BashPatternMatchFrame> {
1161        let Some((harness, pool)) = self.db_harness_and_pool() else {
1162            return Vec::new();
1163        };
1164        let rows = {
1165            let Ok(conn) = pool.lock() else {
1166                return Vec::new();
1167            };
1168            match crate::db::bash_watches::list_bash_pattern_watches_for_session(
1169                &conn, &harness, session_id,
1170            ) {
1171                Ok(rows) => rows,
1172                Err(error) => {
1173                    crate::slog_warn!(
1174                        "failed to load pending bash watches for session {session_id}: {error}"
1175                    );
1176                    return Vec::new();
1177                }
1178            }
1179        };
1180
1181        rows.into_iter()
1182            .filter(|row| row.pending_match)
1183            .filter_map(|row| {
1184                let Some(match_text) = row.match_text else {
1185                    crate::slog_warn!(
1186                        "pending bash watch {}/{} has no match text",
1187                        row.task_id,
1188                        row.watch_id
1189                    );
1190                    return None;
1191                };
1192                let context = row.match_context.unwrap_or_else(|| match_text.clone());
1193                Some(if match_text == WATCH_TARGET_ERASED_TEXT {
1194                    BashPatternMatchFrame::watch_target_erased(
1195                        row.task_id,
1196                        session_id,
1197                        row.watch_id,
1198                        match_text,
1199                        context,
1200                    )
1201                } else if match_text == WATCH_TASK_EXIT_TEXT {
1202                    BashPatternMatchFrame::task_exit(row.task_id, session_id, match_text, context)
1203                } else {
1204                    BashPatternMatchFrame::new(
1205                        row.task_id,
1206                        session_id,
1207                        row.watch_id,
1208                        match_text,
1209                        row.match_offset.unwrap_or_default().max(0) as u64,
1210                        context,
1211                        row.once,
1212                    )
1213                })
1214            })
1215            .collect()
1216    }
1217
1218    fn terminal_db_status_for_session(
1219        &self,
1220        session_id: &str,
1221        task_id: &str,
1222        storage_dir: &Path,
1223    ) -> Option<BgTaskSnapshot> {
1224        let (harness, pool) = self.db_harness_and_pool()?;
1225        let conn = pool.lock().ok()?;
1226        let row =
1227            crate::db::bash_tasks::get_bash_task(&conn, &harness, session_id, task_id).ok()??;
1228        if !task_bundle_is_absent(storage_dir, &row.session_id, &row.task_id) {
1229            return None;
1230        }
1231        let metadata = PersistedTask::from(row.clone());
1232        metadata
1233            .status
1234            .is_terminal()
1235            .then(|| terminal_db_row_snapshot(row, metadata))
1236    }
1237
1238    pub fn has_erased_watch_reference(&self, task_id: &str) -> bool {
1239        self.evaluate_erased_watch_targets();
1240        self.inner
1241            .watch_registry
1242            .lock()
1243            .map(|registry| registry.has_erased_task(task_id))
1244            .unwrap_or(false)
1245    }
1246
1247    pub(crate) fn evaluate_erased_watch_targets(&self) {
1248        let Some((harness, pool)) = self.db_harness_and_pool() else {
1249            return;
1250        };
1251        // The task deletion cascade removes durable watch rows, so only watches
1252        // that are still registered in this process can produce a tombstone.
1253        let watched_task_ids = self
1254            .inner
1255            .watch_registry
1256            .lock()
1257            .map(|registry| registry.watched_task_ids())
1258            .unwrap_or_default();
1259        if watched_task_ids.is_empty() {
1260            return;
1261        }
1262
1263        let candidates = watched_task_ids
1264            .into_iter()
1265            .filter_map(|task_id| {
1266                let task = self.task(&task_id)?;
1267                self.originating_session_has_live_route(&task.session_id)
1268                    .then(|| (task.session_id.clone(), task_id))
1269            })
1270            .collect::<Vec<_>>();
1271        let erased_tasks = {
1272            let Ok(conn) = pool.lock() else {
1273                return;
1274            };
1275            let mut erased_tasks = Vec::new();
1276            for (session_id, task_id) in candidates {
1277                match crate::db::bash_tasks::get_bash_task(&conn, &harness, &session_id, &task_id) {
1278                    Ok(None) => {
1279                        if let Err(error) =
1280                            crate::db::bash_watches::delete_bash_pattern_watches_for_task(
1281                                &conn,
1282                                &harness,
1283                                &session_id,
1284                                &task_id,
1285                            )
1286                        {
1287                            crate::slog_warn!(
1288                                "failed to retire durable watches for erased task {}: {error}",
1289                                task_id
1290                            );
1291                        }
1292                        erased_tasks.push((session_id, task_id));
1293                    }
1294                    Ok(Some(_)) => {}
1295                    Err(error) => {
1296                        crate::slog_warn!(
1297                            "failed to inspect bash watch target {}: {error}",
1298                            task_id
1299                        );
1300                    }
1301                }
1302            }
1303            erased_tasks
1304        };
1305
1306        for (session_id, task_id) in erased_tasks {
1307            let watch_ids = self
1308                .inner
1309                .watch_registry
1310                .lock()
1311                .map(|mut registry| registry.terminalize_erased_task(&task_id))
1312                .unwrap_or_default();
1313            for watch_id in watch_ids {
1314                self.emit_bash_watch_erased(&session_id, &task_id, &watch_id);
1315            }
1316        }
1317    }
1318
1319    fn persist_watch_registration(
1320        &self,
1321        session_id: &str,
1322        task_id: &str,
1323        watch_id: &str,
1324        pattern: &WatchPattern,
1325        once: bool,
1326        stdout_offset: u64,
1327        stderr_offset: u64,
1328        pty_offset: u64,
1329    ) {
1330        let Some((harness, pool)) = self.db_harness_and_pool() else {
1331            return;
1332        };
1333        let Ok(conn) = pool.lock() else {
1334            return;
1335        };
1336        let row = BashPatternWatchRow {
1337            harness,
1338            session_id: session_id.to_string(),
1339            task_id: task_id.to_string(),
1340            watch_id: watch_id.to_string(),
1341            pattern_kind: pattern.kind_name().to_string(),
1342            pattern: pattern.pattern_text().to_string(),
1343            once,
1344            created_at: unix_millis() as i64,
1345            stdout_offset: stdout_offset as i64,
1346            stderr_offset: stderr_offset as i64,
1347            pty_offset: pty_offset as i64,
1348            scanning: true,
1349            pending_match: false,
1350            match_text: None,
1351            match_offset: None,
1352            match_context: None,
1353        };
1354        if let Err(error) = crate::db::bash_watches::upsert_bash_pattern_watch(&conn, &row) {
1355            crate::slog_warn!(
1356                "persist bash_pattern_watch failed for {task_id}/{watch_id}: {error}"
1357            );
1358        }
1359    }
1360
1361    fn delete_persisted_watch(&self, session_id: &str, task_id: &str, watch_id: &str) {
1362        let Some((harness, pool)) = self.db_harness_and_pool() else {
1363            return;
1364        };
1365        let Ok(conn) = pool.lock() else {
1366            return;
1367        };
1368        if let Err(error) = crate::db::bash_watches::delete_bash_pattern_watch(
1369            &conn, &harness, session_id, task_id, watch_id,
1370        ) {
1371            crate::slog_warn!("delete bash_pattern_watch failed for {task_id}/{watch_id}: {error}");
1372        }
1373    }
1374
1375    fn delete_persisted_watches_for_task(&self, session_id: &str, task_id: &str) {
1376        let Some((harness, pool)) = self.db_harness_and_pool() else {
1377            return;
1378        };
1379        let Ok(conn) = pool.lock() else {
1380            return;
1381        };
1382        if let Err(error) = crate::db::bash_watches::delete_bash_pattern_watches_for_task(
1383            &conn, &harness, session_id, task_id,
1384        ) {
1385            crate::slog_warn!("delete bash_pattern_watches for {task_id} failed: {error}");
1386        }
1387    }
1388
1389    fn persist_watch_match(
1390        &self,
1391        session_id: &str,
1392        task_id: &str,
1393        pattern_match: &PatternMatch,
1394        stdout_offset: u64,
1395        stderr_offset: u64,
1396        pty_offset: u64,
1397    ) {
1398        let Some((harness, pool)) = self.db_harness_and_pool() else {
1399            return;
1400        };
1401        let Ok(conn) = pool.lock() else {
1402            return;
1403        };
1404        let Ok(Some(mut row)) = crate::db::bash_watches::get_bash_pattern_watch(
1405            &conn,
1406            &harness,
1407            session_id,
1408            task_id,
1409            &pattern_match.watch_id,
1410        ) else {
1411            return;
1412        };
1413        row.stdout_offset = stdout_offset as i64;
1414        row.stderr_offset = stderr_offset as i64;
1415        row.pty_offset = pty_offset as i64;
1416        row.pending_match = true;
1417        row.match_text = Some(pattern_match.match_text.clone());
1418        row.match_offset = Some(pattern_match.match_offset as i64);
1419        row.match_context = Some(pattern_match.context.clone());
1420        if pattern_match.once {
1421            // Once-watches stop scanning after the first hit but stay durable
1422            // until ack so a lost push can be re-delivered after restart.
1423            row.scanning = false;
1424        }
1425        if let Err(error) = crate::db::bash_watches::upsert_bash_pattern_watch(&conn, &row) {
1426            crate::slog_warn!(
1427                "persist bash_pattern_watch match failed for {}/{}: {error}",
1428                task_id,
1429                pattern_match.watch_id
1430            );
1431        }
1432    }
1433
1434    fn persist_task_watch_cursors(
1435        &self,
1436        session_id: &str,
1437        task_id: &str,
1438        stdout_offset: u64,
1439        stderr_offset: u64,
1440        pty_offset: u64,
1441    ) {
1442        let Some((harness, pool)) = self.db_harness_and_pool() else {
1443            return;
1444        };
1445        let Ok(conn) = pool.lock() else {
1446            return;
1447        };
1448        if let Err(error) = crate::db::bash_watches::update_watch_offsets_for_task(
1449            &conn,
1450            &harness,
1451            session_id,
1452            task_id,
1453            stdout_offset as i64,
1454            stderr_offset as i64,
1455            pty_offset as i64,
1456        ) {
1457            crate::slog_warn!("persist bash_pattern_watch cursors failed for {task_id}: {error}");
1458        }
1459    }
1460
1461    fn watch_stream_cursors(&self, task_id: &str) -> (u64, u64, u64) {
1462        let Ok(registry) = self.inner.watch_registry.lock() else {
1463            return (0, 0, 0);
1464        };
1465        let stdout = registry
1466            .file_cursor(&format!("{task_id}:stdout"))
1467            .unwrap_or(0);
1468        let stderr = registry
1469            .file_cursor(&format!("{task_id}:stderr"))
1470            .unwrap_or(0);
1471        let pty = registry.file_cursor(&format!("{task_id}:pty")).unwrap_or(0);
1472        (stdout, stderr, pty)
1473    }
1474
1475    /// Ack path for pattern watches: once-watches (and any terminal-task watches)
1476    /// are dropped after delivery is confirmed; sticky watches clear pending only.
1477    fn ack_persisted_watches_for_task(&self, session_id: &str, task_id: &str, task_terminal: bool) {
1478        let Some((harness, pool)) = self.db_harness_and_pool() else {
1479            return;
1480        };
1481        let Ok(conn) = pool.lock() else {
1482            return;
1483        };
1484        if task_terminal {
1485            let _ = crate::db::bash_watches::delete_bash_pattern_watches_for_task(
1486                &conn, &harness, session_id, task_id,
1487            );
1488            return;
1489        }
1490        let Ok(rows) = crate::db::bash_watches::list_bash_pattern_watches_for_task(
1491            &conn, &harness, session_id, task_id,
1492        ) else {
1493            return;
1494        };
1495        for mut row in rows {
1496            if row.once && (!row.scanning || row.pending_match) {
1497                let _ = crate::db::bash_watches::delete_bash_pattern_watch(
1498                    &conn,
1499                    &harness,
1500                    session_id,
1501                    task_id,
1502                    &row.watch_id,
1503                );
1504                continue;
1505            }
1506            if row.pending_match {
1507                row.pending_match = false;
1508                row.match_text = None;
1509                row.match_offset = None;
1510                row.match_context = None;
1511                let _ = crate::db::bash_watches::upsert_bash_pattern_watch(&conn, &row);
1512            }
1513        }
1514    }
1515
1516    pub fn record_scanner_report(
1517        &self,
1518        task_id: &str,
1519        session_id: &str,
1520        scanner_report: Vec<PermissionAsk>,
1521    ) -> Result<(), String> {
1522        if scanner_report.is_empty() {
1523            return Ok(());
1524        }
1525        let task = self.task_for_session(task_id, session_id).ok_or_else(|| {
1526            "background task not found while recording scanner report".to_string()
1527        })?;
1528        let metadata = {
1529            let mut state = task
1530                .state
1531                .lock()
1532                .map_err(|_| "background task lock poisoned".to_string())?;
1533            state.metadata.scanner_report = scanner_report;
1534            state.metadata.clone()
1535        };
1536        self.persist_task(&task.paths, &metadata)
1537            .map_err(|error| format!("failed to persist scanner report: {error}"))
1538    }
1539
1540    pub fn configure_long_running_reminders(&self, enabled: bool, interval_ms: u64) {
1541        self.inner
1542            .long_running_reminder_enabled
1543            .store(enabled, Ordering::SeqCst);
1544        self.inner
1545            .long_running_reminder_interval_ms
1546            .store(interval_ms, Ordering::SeqCst);
1547    }
1548
1549    #[cfg(unix)]
1550    #[allow(clippy::too_many_arguments)]
1551    pub fn spawn(
1552        &self,
1553        spawn_plan: SpawnPlan,
1554        command: &str,
1555        session_id: String,
1556        workdir: PathBuf,
1557        env: HashMap<String, String>,
1558        timeout: Option<Duration>,
1559        storage_dir: PathBuf,
1560        max_running: usize,
1561        notify_on_completion: bool,
1562        compressed: bool,
1563        project_root: Option<PathBuf>,
1564    ) -> Result<String, String> {
1565        self.spawn_with_shell(
1566            spawn_plan,
1567            command,
1568            super::BashShell::Bash,
1569            resolve_posix_shell(),
1570            session_id,
1571            workdir,
1572            env,
1573            timeout,
1574            storage_dir,
1575            max_running,
1576            notify_on_completion,
1577            compressed,
1578            project_root,
1579        )
1580    }
1581
1582    #[cfg(unix)]
1583    #[allow(clippy::too_many_arguments)]
1584    pub fn spawn_with_shell(
1585        &self,
1586        spawn_plan: SpawnPlan,
1587        command: &str,
1588        shell: super::BashShell,
1589        shell_path: PathBuf,
1590        session_id: String,
1591        workdir: PathBuf,
1592        #[cfg_attr(not(target_os = "linux"), allow(unused_mut))] mut env: HashMap<String, String>,
1593        timeout: Option<Duration>,
1594        storage_dir: PathBuf,
1595        max_running: usize,
1596        notify_on_completion: bool,
1597        compressed: bool,
1598        project_root: Option<PathBuf>,
1599    ) -> Result<String, String> {
1600        self.start_watchdog();
1601
1602        #[cfg(target_os = "linux")]
1603        let linux_scope = env.remove(LINUX_SCOPE_ENV).as_deref() == Some("1");
1604        #[cfg(not(target_os = "linux"))]
1605        let linux_scope = false;
1606
1607        let running = self.running_count();
1608        if running >= max_running {
1609            #[cfg(unix)]
1610            if let Some(prepared) = spawn_plan.prepared_task() {
1611                let _ = delete_resolved_task(&prepared.resolved_task());
1612            }
1613            return Err(format!(
1614                "background bash task limit exceeded: {running} running (max {max_running})"
1615            ));
1616        }
1617
1618        let timeout = timeout.or(Some(DEFAULT_BG_TIMEOUT));
1619        let timeout_ms = timeout.map(|timeout| timeout.as_millis() as u64);
1620        let (spawn_plan, task_layout) = if let Some(prepared) = spawn_plan.prepared_task() {
1621            (spawn_plan.clone(), prepared.resolved_task())
1622        } else {
1623            let task = allocate_task_layout(&storage_dir, &session_id)
1624                .map_err(|error| format!("failed to create background task layout: {error}"))?;
1625            let root = project_root.as_deref().unwrap_or(&workdir);
1626            let environment =
1627                crate::sandbox_spawn::approved_payload_environment(&env, &std::env::temp_dir());
1628            let prepared = match crate::sandbox_spawn::prepare_task_payload(
1629                &task,
1630                command.as_bytes(),
1631                root,
1632                &workdir,
1633                &crate::sandbox_spawn::AuthenticatedPrincipal::FirstParty,
1634                &shell_path,
1635                &environment,
1636            ) {
1637                Ok(prepared) => prepared,
1638                Err(error) => {
1639                    let _ = delete_resolved_task(&task);
1640                    return Err(error);
1641                }
1642            };
1643            let task = prepared.resolved_task();
1644            (spawn_plan.with_prepared_task(prepared), task)
1645        };
1646        let task_id = task_layout.paths.task_id.clone();
1647        let paths = task_layout.paths.clone();
1648
1649        if self.task(&task_id).is_some() {
1650            let _ = delete_resolved_task(&task_layout);
1651            return Err("background task id collided with a live task".to_string());
1652        }
1653
1654        let mut metadata = PersistedTask::starting(
1655            task_id.clone(),
1656            session_id.clone(),
1657            command.to_string(),
1658            workdir.clone(),
1659            project_root,
1660            timeout_ms,
1661            notify_on_completion,
1662            compressed,
1663        );
1664        // Pipeline-status capture is a Unix-only mechanism: the wrapper needs
1665        // bash/zsh PIPESTATUS and a dedicated inherited fd, neither of which
1666        // exists on the Windows spawn path.
1667        #[cfg(unix)]
1668        let capture_pipeline_status = {
1669            let pipeline = single_top_level_pipeline(command);
1670            let has_pipeline = pipeline.is_some();
1671            let capture = !shell.is_powershell()
1672                && should_capture_pipeline_status(&spawn_plan, has_pipeline, &shell_path);
1673            if !shell.is_powershell() {
1674                metadata.pipeline_segments = pipeline
1675                    .as_ref()
1676                    .map(|pipeline| {
1677                        pipeline
1678                            .segments
1679                            .iter()
1680                            .map(|segment| segment.label.clone())
1681                            .collect()
1682                    })
1683                    .unwrap_or_default();
1684                if has_pipeline && !capture {
1685                    metadata.pipeline_status_unavailable =
1686                        Some(if spawn_plan.is_native_launcher() {
1687                            "native sandbox launcher".to_string()
1688                        } else {
1689                            shell_path
1690                                .file_name()
1691                                .and_then(|name| name.to_str())
1692                                .unwrap_or("unknown shell")
1693                                .to_string()
1694                        });
1695                }
1696            }
1697            capture
1698        };
1699        #[cfg(windows)]
1700        let capture_pipeline_status = false;
1701        attach_sandbox_metadata(&mut metadata, &spawn_plan);
1702        if let Err(error) = write_task_at(&task_layout, &metadata) {
1703            let _ = delete_resolved_task(&task_layout);
1704            return Err(format!(
1705                "failed to persist background task metadata: {error}"
1706            ));
1707        }
1708        self.dual_write_task(&paths, &metadata);
1709
1710        let mut io_handles =
1711            TaskIoHandles::create(&task_layout, BgMode::Pipes, capture_pipeline_status)
1712                .map_err(|error| format!("failed to pre-open task output handles: {error}"))?;
1713        let child = match spawn_detached_child(
1714            &spawn_plan,
1715            command,
1716            shell,
1717            &shell_path,
1718            &paths,
1719            &workdir,
1720            &env,
1721            &mut io_handles,
1722            capture_pipeline_status,
1723            linux_scope,
1724        ) {
1725            Ok(child) => child,
1726            Err(error) => {
1727                crate::slog_warn!("failed to spawn background bash task {task_id}; deleting partial bundle: {error}");
1728                let _ = delete_task_bundle(&paths);
1729                return Err(error);
1730            }
1731        };
1732
1733        let child_pid = child.id();
1734        metadata.mark_running(child_pid, child_pid as i32);
1735        self.persist_task(&paths, &metadata)
1736            .map_err(|e| format!("failed to persist running background task metadata: {e}"))?;
1737
1738        let task = Arc::new(BgTask {
1739            task_id: task_id.clone(),
1740            session_id,
1741            paths: paths.clone(),
1742            artifact_root: canonical_artifact_root(&paths),
1743            started: Instant::now(),
1744            last_reminder_at: Mutex::new(None),
1745            terminal_at: Mutex::new(None),
1746            state: Mutex::new(BgTaskState {
1747                metadata,
1748                runtime: TaskRuntime::Piped(Some(child)),
1749                io_handles: Some(io_handles),
1750                detached: false,
1751                child_exit_observed: false,
1752                descendant_sampling_started: false,
1753                buffer: BgBuffer::registered(&paths, BgMode::Pipes),
1754                terminal_output_cache: None,
1755                pending_terminal_override: None,
1756            }),
1757        });
1758
1759        self.record_live_delivery_session(&task.session_id);
1760        self.inner
1761            .tasks
1762            .lock()
1763            .map_err(|_| "background task registry lock poisoned".to_string())?
1764            .insert(task_id.clone(), task);
1765
1766        Ok(task_id)
1767    }
1768
1769    #[allow(clippy::too_many_arguments)]
1770    pub fn spawn_pty(
1771        &self,
1772        spawn_plan: SpawnPlan,
1773        command: &str,
1774        session_id: String,
1775        workdir: PathBuf,
1776        env: HashMap<String, String>,
1777        timeout: Option<Duration>,
1778        storage_dir: PathBuf,
1779        max_running: usize,
1780        notify_on_completion: bool,
1781        compressed: bool,
1782        project_root: Option<PathBuf>,
1783        rows: u16,
1784        cols: u16,
1785    ) -> Result<String, String> {
1786        self.spawn_pty_with_shell(
1787            spawn_plan,
1788            command,
1789            super::BashShell::Bash,
1790            super::resolve_shell_path(true, super::BashShell::Bash)
1791                .expect("POSIX shell must resolve for bash PTY"),
1792            session_id,
1793            workdir,
1794            env,
1795            timeout,
1796            storage_dir,
1797            max_running,
1798            notify_on_completion,
1799            compressed,
1800            project_root,
1801            rows,
1802            cols,
1803        )
1804    }
1805
1806    #[allow(clippy::too_many_arguments)]
1807    pub fn spawn_pty_with_shell(
1808        &self,
1809        spawn_plan: SpawnPlan,
1810        command: &str,
1811        shell: super::BashShell,
1812        shell_path: PathBuf,
1813        session_id: String,
1814        workdir: PathBuf,
1815        env: HashMap<String, String>,
1816        timeout: Option<Duration>,
1817        storage_dir: PathBuf,
1818        max_running: usize,
1819        notify_on_completion: bool,
1820        compressed: bool,
1821        project_root: Option<PathBuf>,
1822        rows: u16,
1823        cols: u16,
1824    ) -> Result<String, String> {
1825        self.start_watchdog();
1826
1827        let running = self.running_count();
1828        if running >= max_running {
1829            #[cfg(unix)]
1830            if let Some(prepared) = spawn_plan.prepared_task() {
1831                let _ = delete_resolved_task(&prepared.resolved_task());
1832            }
1833            return Err(format!(
1834                "background bash task limit exceeded: {running} running (max {max_running})"
1835            ));
1836        }
1837
1838        let timeout = timeout.or(Some(DEFAULT_BG_TIMEOUT));
1839        let timeout_ms = timeout.map(|timeout| timeout.as_millis() as u64);
1840        #[cfg(unix)]
1841        let (spawn_plan, task_layout) = if let Some(prepared) = spawn_plan.prepared_task() {
1842            (spawn_plan.clone(), prepared.resolved_task())
1843        } else {
1844            let task = allocate_task_layout(&storage_dir, &session_id)
1845                .map_err(|error| format!("failed to create PTY task layout: {error}"))?;
1846            let root = project_root.as_deref().unwrap_or(&workdir);
1847            let environment =
1848                crate::sandbox_spawn::approved_payload_environment(&env, &std::env::temp_dir());
1849            let prepared = match crate::sandbox_spawn::prepare_task_payload(
1850                &task,
1851                command.as_bytes(),
1852                root,
1853                &workdir,
1854                &crate::sandbox_spawn::AuthenticatedPrincipal::FirstParty,
1855                &shell_path,
1856                &environment,
1857            ) {
1858                Ok(prepared) => prepared,
1859                Err(error) => {
1860                    let _ = delete_resolved_task(&task);
1861                    return Err(error);
1862                }
1863            };
1864            let task = prepared.resolved_task();
1865            (spawn_plan.with_prepared_task(prepared), task)
1866        };
1867        #[cfg(windows)]
1868        let task_layout = allocate_task_layout(&storage_dir, &session_id)
1869            .map_err(|error| format!("failed to create PTY task layout: {error}"))?;
1870        let task_id = task_layout.paths.task_id.clone();
1871        let paths = task_layout.paths.clone();
1872
1873        let mut metadata = PersistedTask::starting(
1874            task_id.clone(),
1875            session_id.clone(),
1876            command.to_string(),
1877            workdir.clone(),
1878            project_root,
1879            timeout_ms,
1880            notify_on_completion,
1881            compressed,
1882        );
1883        attach_sandbox_metadata(&mut metadata, &spawn_plan);
1884        metadata.mode = BgMode::Pty;
1885        metadata.pty_rows = Some(rows);
1886        metadata.pty_cols = Some(cols);
1887        if let Err(error) = write_task_at(&task_layout, &metadata) {
1888            let _ = delete_resolved_task(&task_layout);
1889            return Err(format!(
1890                "failed to persist background task metadata: {error}"
1891            ));
1892        }
1893        self.dual_write_task(&paths, &metadata);
1894        let mut io_handles = TaskIoHandles::create(&task_layout, BgMode::Pty, false)
1895            .map_err(|error| format!("failed to pre-open PTY output handles: {error}"))?;
1896
1897        let runtime = match spawn_pty_for_command(
1898            &spawn_plan,
1899            &task_id,
1900            &session_id,
1901            command,
1902            shell,
1903            &shell_path,
1904            &paths,
1905            &workdir,
1906            &env,
1907            rows,
1908            cols,
1909            self.inner.wake_tx.clone(),
1910            &mut io_handles,
1911        ) {
1912            Ok(runtime) => runtime,
1913            Err(error) => {
1914                crate::slog_warn!(
1915                    "failed to spawn PTY background bash task {task_id}; deleting partial bundle: {error}"
1916                );
1917                let _ = delete_task_bundle(&paths);
1918                return Err(error);
1919            }
1920        };
1921
1922        if let Some(child_pid) = runtime.child_pid {
1923            metadata.mark_running(child_pid, child_pid as i32);
1924        } else {
1925            metadata.status = BgTaskStatus::Running;
1926            metadata.pgid = None;
1927        }
1928        self.persist_task(&paths, &metadata)
1929            .map_err(|e| format!("failed to persist running background task metadata: {e}"))?;
1930
1931        let task = Arc::new(BgTask {
1932            task_id: task_id.clone(),
1933            session_id,
1934            paths: paths.clone(),
1935            artifact_root: canonical_artifact_root(&paths),
1936            started: Instant::now(),
1937            last_reminder_at: Mutex::new(None),
1938            terminal_at: Mutex::new(None),
1939            state: Mutex::new(BgTaskState {
1940                metadata,
1941                runtime: TaskRuntime::Pty(Some(runtime)),
1942                io_handles: Some(io_handles),
1943                detached: false,
1944                child_exit_observed: false,
1945                descendant_sampling_started: false,
1946                buffer: BgBuffer::registered(&paths, BgMode::Pty),
1947                terminal_output_cache: None,
1948                pending_terminal_override: None,
1949            }),
1950        });
1951
1952        self.record_live_delivery_session(&task.session_id);
1953        self.inner
1954            .tasks
1955            .lock()
1956            .map_err(|_| "background task registry lock poisoned".to_string())?
1957            .insert(task_id.clone(), task);
1958
1959        Ok(task_id)
1960    }
1961
1962    #[cfg(windows)]
1963    #[allow(clippy::too_many_arguments)]
1964    pub fn spawn(
1965        &self,
1966        spawn_plan: SpawnPlan,
1967        command: &str,
1968        session_id: String,
1969        workdir: PathBuf,
1970        env: HashMap<String, String>,
1971        timeout: Option<Duration>,
1972        storage_dir: PathBuf,
1973        max_running: usize,
1974        notify_on_completion: bool,
1975        compressed: bool,
1976        project_root: Option<PathBuf>,
1977    ) -> Result<String, String> {
1978        self.spawn_with_shell(
1979            spawn_plan,
1980            command,
1981            super::BashShell::Bash,
1982            PathBuf::from("cmd.exe"),
1983            session_id,
1984            workdir,
1985            env,
1986            timeout,
1987            storage_dir,
1988            max_running,
1989            notify_on_completion,
1990            compressed,
1991            project_root,
1992        )
1993    }
1994
1995    #[cfg(windows)]
1996    #[allow(clippy::too_many_arguments)]
1997    pub fn spawn_with_shell(
1998        &self,
1999        spawn_plan: SpawnPlan,
2000        command: &str,
2001        shell: super::BashShell,
2002        shell_path: PathBuf,
2003        session_id: String,
2004        workdir: PathBuf,
2005        env: HashMap<String, String>,
2006        timeout: Option<Duration>,
2007        storage_dir: PathBuf,
2008        max_running: usize,
2009        notify_on_completion: bool,
2010        compressed: bool,
2011        project_root: Option<PathBuf>,
2012    ) -> Result<String, String> {
2013        self.start_watchdog();
2014
2015        let running = self.running_count();
2016        if running >= max_running {
2017            #[cfg(unix)]
2018            if let Some(prepared) = spawn_plan.prepared_task() {
2019                let _ = delete_resolved_task(&prepared.resolved_task());
2020            }
2021            return Err(format!(
2022                "background bash task limit exceeded: {running} running (max {max_running})"
2023            ));
2024        }
2025
2026        let timeout = timeout.or(Some(DEFAULT_BG_TIMEOUT));
2027        let timeout_ms = timeout.map(|timeout| timeout.as_millis() as u64);
2028        let task_layout = allocate_task_layout(&storage_dir, &session_id)
2029            .map_err(|error| format!("failed to create background task layout: {error}"))?;
2030        let task_id = task_layout.paths.task_id.clone();
2031        let paths = task_layout.paths.clone();
2032
2033        let mut metadata = PersistedTask::starting(
2034            task_id.clone(),
2035            session_id.clone(),
2036            command.to_string(),
2037            workdir.clone(),
2038            project_root,
2039            timeout_ms,
2040            notify_on_completion,
2041            compressed,
2042        );
2043        attach_sandbox_metadata(&mut metadata, &spawn_plan);
2044        if let Err(error) = write_task_at(&task_layout, &metadata) {
2045            let _ = delete_resolved_task(&task_layout);
2046            return Err(format!(
2047                "failed to persist background task metadata: {error}"
2048            ));
2049        }
2050        self.dual_write_task(&paths, &metadata);
2051        let mut io_handles = TaskIoHandles::create(&task_layout, BgMode::Pipes, false)
2052            .map_err(|error| format!("failed to pre-open task output handles: {error}"))?;
2053
2054        let child = match spawn_detached_child(
2055            &spawn_plan,
2056            command,
2057            shell,
2058            &shell_path,
2059            &paths,
2060            &workdir,
2061            &env,
2062            &mut io_handles,
2063            false,
2064            false,
2065        ) {
2066            Ok(child) => child,
2067            Err(error) => {
2068                crate::slog_warn!("failed to spawn background bash task {task_id}; deleting partial bundle: {error}");
2069                let _ = delete_task_bundle(&paths);
2070                return Err(error);
2071            }
2072        };
2073
2074        let child_pid = child.id();
2075        metadata.status = BgTaskStatus::Running;
2076        metadata.child_pid = Some(child_pid);
2077        metadata.pgid = None;
2078        self.persist_task(&paths, &metadata)
2079            .map_err(|e| format!("failed to persist running background task metadata: {e}"))?;
2080
2081        let task = Arc::new(BgTask {
2082            task_id: task_id.clone(),
2083            session_id,
2084            paths: paths.clone(),
2085            artifact_root: canonical_artifact_root(&paths),
2086            started: Instant::now(),
2087            last_reminder_at: Mutex::new(None),
2088            terminal_at: Mutex::new(None),
2089            state: Mutex::new(BgTaskState {
2090                metadata,
2091                runtime: TaskRuntime::Piped(Some(child)),
2092                io_handles: Some(io_handles),
2093                detached: false,
2094                child_exit_observed: false,
2095                descendant_sampling_started: false,
2096                buffer: BgBuffer::registered(&paths, BgMode::Pipes),
2097                terminal_output_cache: None,
2098                pending_terminal_override: None,
2099            }),
2100        });
2101
2102        self.record_live_delivery_session(&task.session_id);
2103        self.inner
2104            .tasks
2105            .lock()
2106            .map_err(|_| "background task registry lock poisoned".to_string())?
2107            .insert(task_id.clone(), task);
2108
2109        Ok(task_id)
2110    }
2111
2112    pub fn write_pty(
2113        &self,
2114        task_id: &str,
2115        session_id: &str,
2116        input: &[u8],
2117    ) -> Result<usize, String> {
2118        let task = self
2119            .task_for_session(task_id, session_id)
2120            .ok_or_else(|| "task_not_found".to_string())?;
2121
2122        let writer = {
2123            let state = task
2124                .state
2125                .lock()
2126                .map_err(|_| "background task lock poisoned".to_string())?;
2127            if state.metadata.mode != BgMode::Pty {
2128                return Err("task_not_pty".to_string());
2129            }
2130            if state.metadata.status.is_terminal() {
2131                return Err("task_exited".to_string());
2132            }
2133            match &state.runtime {
2134                TaskRuntime::Pty(Some(runtime)) => Arc::clone(&runtime.writer),
2135                TaskRuntime::Pty(None) => return Err("task_exited".to_string()),
2136                TaskRuntime::Piped(_) => return Err("task_not_pty".to_string()),
2137            }
2138        };
2139
2140        let mut writer = writer
2141            .lock()
2142            .map_err(|_| "PTY writer lock poisoned".to_string())?;
2143        writer
2144            .write_all(input)
2145            .map_err(|error| format!("failed to write to PTY: {error}"))?;
2146        writer
2147            .flush()
2148            .map_err(|error| format!("failed to flush PTY writer: {error}"))?;
2149        Ok(input.len())
2150    }
2151
2152    pub fn replay_session(&self, storage_dir: &Path, session_id: &str) -> Result<(), String> {
2153        self.replay_session_inner(storage_dir, session_id, None)
2154    }
2155
2156    /// Mark `session_id`'s task directory as under replay recovery until the
2157    /// returned guard is dropped.
2158    ///
2159    /// Recovery rewrites the metadata of every task it restores, so for that
2160    /// window the directory's contents are mid-flight rather than settled. The
2161    /// persisted GC consults this before it deletes or quarantines anything, so
2162    /// a sweep cannot destroy a task the recovery just put back.
2163    #[doc(hidden)]
2164    pub fn begin_session_recovery(
2165        &self,
2166        storage_dir: &Path,
2167        session_id: &str,
2168    ) -> SessionRecoveryGuard {
2169        let key = session_recovery_key(storage_dir, session_id);
2170        if let Some(key) = key.as_ref() {
2171            self.inner
2172                .session_recovery
2173                .lock()
2174                .unwrap_or_else(std::sync::PoisonError::into_inner)
2175                .entry(key.clone())
2176                .or_default()
2177                .in_flight += 1;
2178        }
2179        SessionRecoveryGuard {
2180            registry: self.clone(),
2181            key,
2182        }
2183    }
2184
2185    /// Recovery activity recorded for a session task directory, by its name.
2186    fn session_recovery_state(&self, session_dir: &Path) -> SessionRecovery {
2187        let Some(key) = session_dir.file_name().and_then(|name| name.to_str()) else {
2188            return SessionRecovery::default();
2189        };
2190        self.inner
2191            .session_recovery
2192            .lock()
2193            .unwrap_or_else(std::sync::PoisonError::into_inner)
2194            .get(key)
2195            .copied()
2196            .unwrap_or_default()
2197    }
2198
2199    /// Thread name recorded by the last `maybe_gc_persisted` run, if any.
2200    /// Recorded as the run exits, so `Some` also means that run has finished.
2201    #[doc(hidden)]
2202    pub fn persisted_gc_thread(&self) -> Option<String> {
2203        self.inner
2204            .persisted_gc_thread
2205            .lock()
2206            .unwrap_or_else(std::sync::PoisonError::into_inner)
2207            .clone()
2208    }
2209
2210    pub fn replay_session_for_project(
2211        &self,
2212        storage_dir: &Path,
2213        session_id: &str,
2214        project_root: &Path,
2215    ) -> Result<(), String> {
2216        self.replay_session_inner(storage_dir, session_id, Some(project_root))
2217    }
2218
2219    fn retire_orphaned_watch_tombstones(&self, binding_session_id: &str) -> Result<(), String> {
2220        let Some((harness, pool)) = self.db_harness_and_pool() else {
2221            return Ok(());
2222        };
2223        let retired = {
2224            let conn = pool
2225                .lock()
2226                .map_err(|_| "background task database lock poisoned".to_string())?;
2227            let rows = crate::db::bash_watches::list_bash_pattern_watches(&conn, &harness)
2228                .map_err(|error| format!("failed to inspect persisted bash watches: {error}"))?;
2229            let mut retired = Vec::new();
2230            for row in rows {
2231                let task_exists = match crate::db::bash_tasks::get_bash_task(
2232                    &conn,
2233                    &harness,
2234                    &row.session_id,
2235                    &row.task_id,
2236                ) {
2237                    Ok(task) => task.is_some(),
2238                    // Fail-safe retention on transient DB error: destructive retirement
2239                    // requires confirmed absence (Ok(None)). Treating a transient SQLite
2240                    // read error as absent would permanently delete active watches on lock
2241                    // contention; defer retirement until a clean read verifies absence.
2242                    Err(_) => true,
2243                };
2244                let is_erased_target = !task_exists
2245                    || (row.pending_match
2246                        && row.match_text.as_deref() == Some(WATCH_TARGET_ERASED_TEXT));
2247                if !is_erased_target
2248                    || !self.should_retire_foreign_delivery(&row.session_id, binding_session_id)
2249                {
2250                    continue;
2251                }
2252                let deleted = crate::db::bash_watches::delete_bash_pattern_watch(
2253                    &conn,
2254                    &harness,
2255                    &row.session_id,
2256                    &row.task_id,
2257                    &row.watch_id,
2258                )
2259                .map_err(|error| {
2260                    format!(
2261                        "failed to retire orphaned bash watch {}/{}: {error}",
2262                        row.task_id, row.watch_id
2263                    )
2264                })?;
2265                if deleted > 0 {
2266                    retired.push((row.session_id, row.task_id));
2267                }
2268            }
2269            retired
2270        };
2271
2272        for (originating_session_id, task_id) in retired {
2273            self.clear_task_watch_state(&task_id);
2274            if let Ok(mut registry) = self.inner.watch_registry.lock() {
2275                registry.forget_erased_task(&task_id);
2276            }
2277            crate::slog_info!(
2278                "retired orphaned background watch tombstone: task_id={} originating_session={}",
2279                task_id,
2280                originating_session_id
2281            );
2282        }
2283        Ok(())
2284    }
2285
2286    fn retire_already_reaped_orphaned_completion(
2287        &self,
2288        originating_session_id: &str,
2289        task_id: &str,
2290    ) {
2291        if let Some((harness, pool)) = self.db_harness_and_pool() {
2292            match pool.lock() {
2293                Ok(conn) => {
2294                    if let Err(error) = crate::db::bash_tasks::delete_bash_task(
2295                        &conn,
2296                        &harness,
2297                        originating_session_id,
2298                        task_id,
2299                    ) {
2300                        crate::slog_warn!(
2301                            "failed to delete already-reaped orphaned background completion row: task_id={task_id} error={error}"
2302                        );
2303                    }
2304                }
2305                Err(_) => crate::slog_warn!(
2306                    "failed to delete already-reaped orphaned background completion row: task_id={task_id} error=database_lock_poisoned"
2307                ),
2308            }
2309        }
2310        let _ = self.remove_pending_completion(task_id);
2311        self.ack_persisted_watches_for_task(originating_session_id, task_id, true);
2312        crate::slog_warn!(
2313            "orphaned completion already reaped: task_id={task_id} reason=layout_missing"
2314        );
2315    }
2316
2317    fn retire_rehydrated_orphaned_completion(
2318        &self,
2319        task: &Arc<BgTask>,
2320        binding_session_id: &str,
2321    ) -> Result<(), String> {
2322        let should_retire = task
2323            .state
2324            .lock()
2325            .map_err(|_| "background task lock poisoned".to_string())
2326            .map(|state| {
2327                state.metadata.status.is_terminal()
2328                    && !state.metadata.completion_delivered
2329                    && (self.should_retire_foreign_delivery(&task.session_id, binding_session_id)
2330                        || !task.paths.dir.exists())
2331            })?;
2332        if !should_retire {
2333            return Ok(());
2334        }
2335
2336        match task.set_completion_delivered(true, self) {
2337            Ok(()) => {
2338                let _ = self.remove_pending_completion(&task.task_id);
2339                self.ack_persisted_watches_for_task(&task.session_id, &task.task_id, true);
2340                crate::slog_info!(
2341                    "retired orphaned background completion: task_id={} originating_session={}",
2342                    task.task_id,
2343                    task.session_id
2344                );
2345            }
2346            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
2347                if let Ok(mut state) = task.state.lock() {
2348                    state.metadata.completion_delivered = true;
2349                }
2350                self.retire_already_reaped_orphaned_completion(&task.session_id, &task.task_id);
2351            }
2352            Err(error) => {
2353                return Err(format!("failed to update completion delivery: {error}"));
2354            }
2355        }
2356        Ok(())
2357    }
2358
2359    fn enqueue_replay_completion_if_needed(
2360        &self,
2361        metadata: &mut PersistedTask,
2362        paths: &TaskPaths,
2363        binding_session_id: &str,
2364    ) -> Result<(), String> {
2365        if metadata.status.is_terminal()
2366            && !metadata.completion_delivered
2367            && self.should_retire_foreign_delivery(&metadata.session_id, binding_session_id)
2368        {
2369            metadata.completion_delivered = true;
2370            self.persist_task(paths, metadata).map_err(|error| {
2371                format!(
2372                    "failed to retire orphaned background completion {}: {error}",
2373                    metadata.task_id
2374                )
2375            })?;
2376            let _ = self.remove_pending_completion(&metadata.task_id);
2377            self.ack_persisted_watches_for_task(&metadata.session_id, &metadata.task_id, true);
2378            crate::slog_info!(
2379                "retired orphaned background completion: task_id={} originating_session={}",
2380                metadata.task_id,
2381                metadata.session_id
2382            );
2383            return Ok(());
2384        }
2385
2386        self.enqueue_completion_if_needed(metadata, Some(paths), false);
2387        Ok(())
2388    }
2389
2390    fn replay_session_inner(
2391        &self,
2392        storage_dir: &Path,
2393        session_id: &str,
2394        project_root: Option<&Path>,
2395    ) -> Result<(), String> {
2396        // Held for the whole call: from here on this session's task directory is
2397        // being rewritten, and the persisted GC below sweeps the same storage
2398        // root from another thread. Tasks belonging to other sessions are
2399        // guarded as they are recovered, once the replay set is known.
2400        let mut recovering = vec![self.begin_session_recovery(storage_dir, session_id)];
2401        // Standalone NDJSON: every session for a project replays through the same
2402        // bridge, so replay unions this session into live delivery routes.
2403        // Only subc route synchronization (subc/mod.rs:2120) replaces the whole set.
2404        self.record_live_delivery_session(session_id);
2405        self.retire_orphaned_watch_tombstones(session_id)?;
2406        self.start_watchdog();
2407        if !self.inner.persisted_gc_started.swap(true, Ordering::SeqCst) {
2408            // The persisted GC walks every session under the shared storage root
2409            // (liveness probes, row deletes, quarantines) and scales with the
2410            // machine's task history, not this session's. Replay runs before the
2411            // first request after configure, so keeping the GC inline made that
2412            // request wait on storage-wide housekeeping: 2.4 s on a warm box,
2413            // past the plugin's 5 s request timeout under load. Replay itself
2414            // needs only this project's rows, so the GC runs detached; a row it
2415            // would have deleted is at worst rehydrated as terminal and reaped
2416            // by the watchdog. Detaching it means it walks the same directories
2417            // this replay is rewriting, so the recovery guard above is what
2418            // keeps the two apart.
2419            let registry = self.clone();
2420            let storage_dir = storage_dir.to_path_buf();
2421            let spawned = std::thread::Builder::new()
2422                .name("aft-bash-task-gc".to_string())
2423                .spawn(move || {
2424                    if let Err(error) = registry.maybe_gc_persisted(&storage_dir) {
2425                        crate::slog_warn!("failed to GC persisted background bash tasks: {error}");
2426                    }
2427                });
2428            if let Err(error) = spawned {
2429                crate::slog_warn!("failed to spawn persisted background task GC: {error}");
2430            }
2431        }
2432
2433        let canonical_project = project_root.map(canonicalized_path);
2434        // Replay strategy: DB is the post-v0.27 source of truth. Disk
2435        // fallback handles pre-v0.27 tasks that haven't been migrated and
2436        // the cold-start `__default__` namespace (configure runs before any
2437        // user session exists, so plugin-init triggers a session-less DB
2438        // lookup that will be empty until a real session writes a task).
2439        //
2440        // We deliberately keep the empty-DB / empty-disk path silent — it's
2441        // the normal startup case and would otherwise fire on every configure
2442        // (see GitHub user report against v0.27.0). INFO-level logs only when
2443        // disk actually returned tasks (real migration signal); WARN when the
2444        // DB lookup itself errored.
2445        let tasks = match self.replay_session_from_db(session_id, project_root) {
2446            Some(Ok(tasks)) if !tasks.is_empty() => tasks,
2447            Some(Ok(_)) => {
2448                let disk_tasks = self.replay_session_from_disk(storage_dir, session_id)?;
2449                if !disk_tasks.is_empty() {
2450                    crate::slog_info!(
2451                        "bash task replay: 0 in DB for session {}, {} from disk fallback",
2452                        session_id,
2453                        disk_tasks.len()
2454                    );
2455                }
2456                disk_tasks
2457            }
2458            Some(Err(error)) => {
2459                crate::slog_warn!(
2460                    "bash task replay DB lookup failed for session {}; falling back to disk: {}",
2461                    session_id,
2462                    error
2463                );
2464                self.replay_session_from_disk(storage_dir, session_id)?
2465            }
2466            None => {
2467                // DB pool unconfigured — common in tests + before harness is set.
2468                self.replay_session_from_disk(storage_dir, session_id)?
2469            }
2470        };
2471
2472        // A project-scoped replay restores tasks from sessions other than the
2473        // one being bound, so guard those directories too before the loop
2474        // touches them.
2475        let mut guarded = HashSet::from([session_id.to_string()]);
2476        for metadata in &tasks {
2477            if guarded.insert(metadata.session_id.clone()) {
2478                recovering.push(self.begin_session_recovery(storage_dir, &metadata.session_id));
2479            }
2480        }
2481
2482        for mut metadata in tasks {
2483            if project_root.is_none() && metadata.session_id != session_id {
2484                continue;
2485            }
2486            if let Some(canonical_project) = canonical_project.as_deref() {
2487                let metadata_project = metadata.project_root.as_deref().map(canonicalized_path);
2488                if metadata_project.as_deref() != Some(canonical_project) {
2489                    continue;
2490                }
2491            }
2492
2493            if validate_task_id(&metadata.task_id).is_err() {
2494                crate::slog_warn!(
2495                    "ignoring persisted background task with invalid id {:?}",
2496                    metadata.task_id
2497                );
2498                continue;
2499            }
2500            // Another session in this daemon may bind the same project. Keep the
2501            // authoritative child handle and pinned artifact handles already in memory;
2502            // replacing them with a disk-only replay would orphan control of a live task.
2503            if let Some(task) = self.task(&metadata.task_id) {
2504                self.retire_rehydrated_orphaned_completion(&task, session_id)?;
2505                continue;
2506            }
2507            let session_dir = session_tasks_dir(storage_dir, &metadata.session_id);
2508            let resolved = match resolve_task_layout(&session_dir, &metadata.task_id) {
2509                Ok(task) => task,
2510                Err(error) => {
2511                    if error.kind() == std::io::ErrorKind::NotFound
2512                        && !Self::persisted_task_process_is_alive(&metadata)
2513                        && (self.should_retire_foreign_delivery(&metadata.session_id, session_id)
2514                            || &metadata.session_id != session_id)
2515                    {
2516                        self.retire_already_reaped_orphaned_completion(
2517                            &metadata.session_id,
2518                            &metadata.task_id,
2519                        );
2520                        continue;
2521                    }
2522                    if Self::persisted_task_process_is_alive(&metadata) {
2523                        crate::slog_warn!(
2524                            "refusing to quarantine unresolved live background task {}: {error}",
2525                            metadata.task_id
2526                        );
2527                        continue;
2528                    }
2529                    crate::slog_warn!(
2530                        "quarantining unresolved background task {}: {error}",
2531                        metadata.task_id
2532                    );
2533                    let _ = quarantine_task_layout(
2534                        storage_dir,
2535                        &session_dir,
2536                        &metadata.task_id,
2537                        "invalid",
2538                    );
2539                    continue;
2540                }
2541            };
2542            match read_task_at(&resolved) {
2543                Ok(disk)
2544                    if disk.task_id == metadata.task_id
2545                        && disk.session_id == metadata.session_id => {}
2546                Ok(_) | Err(_) => {
2547                    if Self::persisted_task_process_is_alive(&metadata) {
2548                        crate::slog_warn!(
2549                            "refusing to quarantine mismatched live background task {}",
2550                            metadata.task_id
2551                        );
2552                        continue;
2553                    }
2554                    let _ = quarantine_task_layout(
2555                        storage_dir,
2556                        &session_dir,
2557                        &metadata.task_id,
2558                        "mismatch",
2559                    );
2560                    continue;
2561                }
2562            }
2563            let paths = resolved.paths;
2564            match metadata.status {
2565                BgTaskStatus::Starting => {
2566                    let completion_was_delivered = metadata.completion_delivered;
2567                    metadata.mark_terminal(
2568                        BgTaskStatus::Failed,
2569                        None,
2570                        Some("spawn aborted".to_string()),
2571                    );
2572                    metadata.completion_delivered |= completion_was_delivered;
2573                    let _ = self.persist_task(&paths, &metadata);
2574                    self.enqueue_replay_completion_if_needed(&mut metadata, &paths, session_id)?;
2575                    self.insert_rehydrated_task(metadata, paths, true)?;
2576                }
2577                BgTaskStatus::Running | BgTaskStatus::Killing => {
2578                    if metadata.mode == BgMode::Pty {
2579                        if let Ok(Some(marker)) = read_exit_marker(&paths) {
2580                            let completion_was_delivered = metadata.completion_delivered;
2581                            metadata = terminal_metadata_from_marker(metadata, marker, None);
2582                            metadata.completion_delivered |= completion_was_delivered;
2583                            let _ = self.persist_task(&paths, &metadata);
2584                            self.enqueue_replay_completion_if_needed(
2585                                &mut metadata,
2586                                &paths,
2587                                session_id,
2588                            )?;
2589                            self.insert_rehydrated_task(metadata, paths, true)?;
2590                        } else if metadata.status.is_terminal() {
2591                            self.insert_rehydrated_task(metadata, paths, true)?;
2592                        } else {
2593                            let completion_was_delivered = metadata.completion_delivered;
2594                            metadata.mark_terminal(
2595                                BgTaskStatus::Killed,
2596                                None,
2597                                Some("pty_lost_on_bridge_restart".to_string()),
2598                            );
2599                            metadata.completion_delivered |= completion_was_delivered;
2600                            let _ = self.persist_task(&paths, &metadata);
2601                            self.enqueue_replay_completion_if_needed(
2602                                &mut metadata,
2603                                &paths,
2604                                session_id,
2605                            )?;
2606                            self.insert_rehydrated_task(metadata, paths, true)?;
2607                        }
2608                    } else if let Ok(Some(marker)) = read_exit_marker(&paths) {
2609                        let reason = (metadata.status == BgTaskStatus::Killing).then(|| {
2610                            "recovered from inconsistent killing state on replay".to_string()
2611                        });
2612                        if reason.is_some() {
2613                            crate::slog_warn!("background task {} had killing state with exit marker; preferring marker",
2614                            metadata.task_id);
2615                        }
2616                        let completion_was_delivered = metadata.completion_delivered;
2617                        metadata = terminal_metadata_from_marker(metadata, marker, reason);
2618                        metadata.completion_delivered |= completion_was_delivered;
2619                        let _ = self.persist_task(&paths, &metadata);
2620                        self.enqueue_replay_completion_if_needed(
2621                            &mut metadata,
2622                            &paths,
2623                            session_id,
2624                        )?;
2625                        self.insert_rehydrated_task(metadata, paths, true)?;
2626                    } else if metadata.status == BgTaskStatus::Killing {
2627                        let _ = write_kill_marker_if_absent(&paths);
2628                        let completion_was_delivered = metadata.completion_delivered;
2629                        metadata.mark_terminal(
2630                            BgTaskStatus::Killed,
2631                            None,
2632                            Some("recovered from inconsistent killing state on replay".to_string()),
2633                        );
2634                        metadata.completion_delivered |= completion_was_delivered;
2635                        let _ = self.persist_task(&paths, &metadata);
2636                        self.enqueue_replay_completion_if_needed(
2637                            &mut metadata,
2638                            &paths,
2639                            session_id,
2640                        )?;
2641                        self.insert_rehydrated_task(metadata, paths, true)?;
2642                    } else if Self::persisted_task_process_is_alive(&metadata) {
2643                        self.insert_rehydrated_task(metadata, paths, true)?;
2644                    } else {
2645                        let completion_was_delivered = metadata.completion_delivered;
2646                        metadata.mark_terminal(
2647                            BgTaskStatus::FateUnknown,
2648                            None,
2649                            Some(restart_fate_unknown_reason(&metadata, &paths)),
2650                        );
2651                        metadata.completion_delivered |= completion_was_delivered;
2652                        let _ = self.persist_task(&paths, &metadata);
2653                        self.enqueue_replay_completion_if_needed(
2654                            &mut metadata,
2655                            &paths,
2656                            session_id,
2657                        )?;
2658                        self.insert_rehydrated_task(metadata, paths, true)?;
2659                    }
2660                }
2661                _ if metadata.status.is_terminal() => {
2662                    // Borrow `paths` for the completion enqueue BEFORE
2663                    // `insert_rehydrated_task` consumes it. The completion
2664                    // helper only reads from `paths` (stdout/stderr/exit) to
2665                    // reconstruct a tail preview, so it must see the same
2666                    // paths the rehydrated task will own.
2667                    self.enqueue_replay_completion_if_needed(&mut metadata, &paths, session_id)?;
2668                    self.insert_rehydrated_task(metadata, paths, true)?;
2669                }
2670                _ => {}
2671            }
2672        }
2673
2674        // Every recovery write for this replay has landed; release the
2675        // directories so the sweep can judge them as settled storage.
2676        drop(recovering);
2677        Ok(())
2678    }
2679
2680    fn replay_session_from_db(
2681        &self,
2682        session_id: &str,
2683        project_root: Option<&Path>,
2684    ) -> Option<Result<Vec<PersistedTask>, String>> {
2685        let pool = self
2686            .inner
2687            .db_pool
2688            .read()
2689            .ok()
2690            .and_then(|slot| slot.clone())?;
2691        let harness = self
2692            .inner
2693            .db_harness
2694            .read()
2695            .ok()
2696            .and_then(|slot| slot.clone())?;
2697        let conn = match pool.lock() {
2698            Ok(conn) => conn,
2699            Err(_) => return Some(Err("db mutex poisoned".to_string())),
2700        };
2701        let rows = if let Some(project_root) = project_root {
2702            let project_key = crate::path_identity::project_scope_key(project_root);
2703            crate::db::bash_tasks::list_replayable_bash_tasks_for_project(
2704                &conn,
2705                &harness,
2706                &project_key,
2707            )
2708        } else {
2709            crate::db::bash_tasks::list_bash_tasks_for_session(&conn, &harness, session_id)
2710        };
2711        Some(
2712            rows.map(|rows| rows.into_iter().map(PersistedTask::from).collect())
2713                .map_err(|error| error.to_string()),
2714        )
2715    }
2716
2717    fn replay_session_from_disk(
2718        &self,
2719        storage_dir: &Path,
2720        session_id: &str,
2721    ) -> Result<Vec<PersistedTask>, String> {
2722        let dir = session_tasks_dir(storage_dir, session_id);
2723        if !dir.exists() {
2724            return Ok(Vec::new());
2725        }
2726
2727        let (task_ids, invalid_entries) = discover_task_ids(&dir)
2728            .map_err(|error| format!("failed to discover background task layouts: {error}"))?;
2729        for entry in invalid_entries {
2730            if let Err(error) = quarantine_invalid_entry(storage_dir, &dir, &entry) {
2731                crate::slog_warn!(
2732                    "failed to quarantine invalid background task entry {:?}: {error}",
2733                    entry
2734                );
2735            }
2736        }
2737
2738        let mut tasks = Vec::new();
2739        for task_id in task_ids {
2740            let task = match resolve_task_layout(&dir, &task_id) {
2741                Ok(task) => task,
2742                Err(error)
2743                    if error.kind() == std::io::ErrorKind::NotFound
2744                        && uninitialized_layout_is_recent(
2745                            &dir,
2746                            &task_id,
2747                            Duration::from_secs(5 * 60),
2748                        )
2749                        .unwrap_or(false) =>
2750                {
2751                    continue;
2752                }
2753                Err(error) => {
2754                    if self.db_has_live_process_for_task(&task_id) {
2755                        crate::slog_warn!(
2756                            "refusing to quarantine unresolved live background task {task_id} during replay: {error}"
2757                        );
2758                        continue;
2759                    }
2760                    crate::slog_warn!(
2761                        "quarantining unresolved background task {task_id} during replay: {error}"
2762                    );
2763                    let _ = quarantine_task_layout(storage_dir, &dir, &task_id, "invalid");
2764                    continue;
2765                }
2766            };
2767            match read_task_at(&task) {
2768                Ok(metadata) if metadata.session_id == session_id => tasks.push(metadata),
2769                Ok(_) => {
2770                    crate::slog_warn!(
2771                        "quarantining background task {task_id} with mismatched session metadata"
2772                    );
2773                    let _ = quarantine_task_layout(storage_dir, &dir, &task_id, "mismatch");
2774                }
2775                Err(error) => {
2776                    if self.db_has_live_process_for_task(&task_id) {
2777                        crate::slog_warn!(
2778                            "refusing to quarantine unreadable live background task {task_id} during replay: {error}"
2779                        );
2780                        continue;
2781                    }
2782                    crate::slog_warn!(
2783                        "quarantining invalid background task metadata {task_id} during replay: {error}"
2784                    );
2785                    let _ = quarantine_task_layout(storage_dir, &dir, &task_id, "invalid");
2786                }
2787            }
2788        }
2789        Ok(tasks)
2790    }
2791
2792    pub fn register_watch(
2793        &self,
2794        task_id: String,
2795        pattern: WatchPattern,
2796        once: bool,
2797    ) -> Result<String, &'static str> {
2798        let task = self.task(&task_id).ok_or("task_not_found")?;
2799        self.record_live_delivery_session(&task.session_id);
2800        validate_task_id(&task_id).map_err(|_| "invalid_task_id")?;
2801        let (mode, terminal_at_registration) = task
2802            .state
2803            .lock()
2804            .map(|state| {
2805                (
2806                    state.metadata.mode.clone(),
2807                    state.metadata.status.is_terminal(),
2808                )
2809            })
2810            .map_err(|_| "background_task_lock_poisoned")?;
2811        let mut stdout = (mode == BgMode::Pipes)
2812            .then(|| open_task_artifact(&task.paths, TaskArtifact::Stdout))
2813            .transpose()
2814            .map_err(|_| "artifact_refused")?;
2815        let mut stderr = (mode == BgMode::Pipes)
2816            .then(|| open_task_artifact(&task.paths, TaskArtifact::Stderr))
2817            .transpose()
2818            .map_err(|_| "artifact_refused")?;
2819        let mut pty = (mode == BgMode::Pty)
2820            .then(|| open_task_artifact(&task.paths, TaskArtifact::Pty))
2821            .transpose()
2822            .map_err(|_| "artifact_refused")?;
2823
2824        let mut terminal_matches = Vec::new();
2825        let scanned_terminal = terminal_at_registration;
2826        let watch_id = {
2827            let mut registry = self
2828                .inner
2829                .watch_registry
2830                .lock()
2831                .map_err(|_| "watch_registry_poisoned")?;
2832            let watch_id = registry.register(task_id.clone(), pattern.clone(), once)?;
2833            match &mode {
2834                BgMode::Pipes => {
2835                    let stdout_key = format!("{task_id}:stdout");
2836                    let stderr_key = format!("{task_id}:stderr");
2837                    if terminal_at_registration {
2838                        registry.set_file_cursor(&stdout_key, 0);
2839                        registry.set_file_cursor(&stderr_key, 0);
2840                        terminal_matches.extend(registry.scan_file_new_bytes(
2841                            &stdout_key,
2842                            &task_id,
2843                            stdout.as_mut().expect("pipe stdout opened"),
2844                        ));
2845                        terminal_matches.extend(registry.scan_file_new_bytes(
2846                            &stderr_key,
2847                            &task_id,
2848                            stderr.as_mut().expect("pipe stderr opened"),
2849                        ));
2850                    } else {
2851                        registry.prime_file_cursor(
2852                            &stdout_key,
2853                            stdout.as_ref().expect("pipe stdout opened"),
2854                        );
2855                        registry.prime_file_cursor(
2856                            &stderr_key,
2857                            stderr.as_ref().expect("pipe stderr opened"),
2858                        );
2859                    }
2860                }
2861                BgMode::Pty => {
2862                    let pty_key = format!("{task_id}:pty");
2863                    if terminal_at_registration {
2864                        registry.set_file_cursor(&pty_key, 0);
2865                        terminal_matches.extend(registry.scan_file_new_bytes(
2866                            &pty_key,
2867                            &task_id,
2868                            pty.as_mut().expect("PTY artifact opened"),
2869                        ));
2870                    } else {
2871                        registry.prime_file_cursor(
2872                            &pty_key,
2873                            pty.as_ref().expect("PTY artifact opened"),
2874                        );
2875                    }
2876                }
2877            }
2878            watch_id
2879        };
2880
2881        let (stdout_offset, stderr_offset, pty_offset) = self.watch_stream_cursors(&task_id);
2882        self.persist_watch_registration(
2883            &task.session_id,
2884            &task_id,
2885            &watch_id,
2886            &pattern,
2887            once,
2888            stdout_offset,
2889            stderr_offset,
2890            pty_offset,
2891        );
2892
2893        if task.is_terminal() {
2894            if !scanned_terminal {
2895                terminal_matches = {
2896                    let mut registry = self
2897                        .inner
2898                        .watch_registry
2899                        .lock()
2900                        .map_err(|_| "watch_registry_poisoned")?;
2901                    match &mode {
2902                        BgMode::Pipes => {
2903                            let stdout_key = format!("{task_id}:stdout");
2904                            let stderr_key = format!("{task_id}:stderr");
2905                            registry.set_file_cursor(&stdout_key, 0);
2906                            registry.set_file_cursor(&stderr_key, 0);
2907                            let mut matches = registry.scan_file_new_bytes(
2908                                &stdout_key,
2909                                &task_id,
2910                                stdout.as_mut().expect("pipe stdout opened"),
2911                            );
2912                            matches.extend(registry.scan_file_new_bytes(
2913                                &stderr_key,
2914                                &task_id,
2915                                stderr.as_mut().expect("pipe stderr opened"),
2916                            ));
2917                            matches
2918                        }
2919                        BgMode::Pty => {
2920                            let pty_key = format!("{task_id}:pty");
2921                            registry.set_file_cursor(&pty_key, 0);
2922                            registry.scan_file_new_bytes(
2923                                &pty_key,
2924                                &task_id,
2925                                pty.as_mut().expect("PTY artifact opened"),
2926                            )
2927                        }
2928                    }
2929                };
2930            }
2931
2932            let (stdout_offset, stderr_offset, pty_offset) = self.watch_stream_cursors(&task_id);
2933            let (watch_controlled, watch_matched) = self.task_watch_state(&task_id);
2934            if terminal_matches.is_empty() && (!watch_controlled || watch_matched) {
2935                if watch_matched {
2936                    self.clear_task_watch_state(&task_id);
2937                }
2938                return Ok(watch_id);
2939            }
2940
2941            let completion = self
2942                .remove_pending_completion(&task_id)
2943                .or_else(|| self.completion_snapshot_for_task(&task));
2944            if terminal_matches.is_empty() {
2945                if let Some(completion) = completion.as_ref() {
2946                    self.record_bash_watch_exit(completion, true);
2947                }
2948            } else {
2949                for pattern_match in &terminal_matches {
2950                    self.persist_watch_match(
2951                        &task.session_id,
2952                        &task_id,
2953                        pattern_match,
2954                        stdout_offset,
2955                        stderr_offset,
2956                        pty_offset,
2957                    );
2958                    self.emit_bash_pattern_match(&task.session_id, pattern_match.clone());
2959                }
2960            }
2961            self.clear_task_watch_state(&task_id);
2962        }
2963
2964        Ok(watch_id)
2965    }
2966
2967    pub fn unregister_watch(&self, task_id: &str, watch_id: &str) {
2968        let session_id = self.task(task_id).map(|task| task.session_id.clone());
2969        if let Ok(mut registry) = self.inner.watch_registry.lock() {
2970            registry.unregister(task_id, watch_id);
2971        }
2972        if let Some(session_id) = session_id {
2973            self.delete_persisted_watch(&session_id, task_id, watch_id);
2974        }
2975    }
2976
2977    pub fn active_watch_count(&self, task_id: &str) -> usize {
2978        self.inner
2979            .watch_registry
2980            .lock()
2981            .map(|registry| registry.active_count(task_id))
2982            .unwrap_or(0)
2983    }
2984
2985    fn task_watch_state(&self, task_id: &str) -> (bool, bool) {
2986        self.inner
2987            .watch_registry
2988            .lock()
2989            .map(|registry| {
2990                (
2991                    registry.has_controlled_task(task_id),
2992                    registry.has_matched_task(task_id),
2993                )
2994            })
2995            .unwrap_or((false, false))
2996    }
2997
2998    fn task_has_watch_control(&self, task_id: &str) -> bool {
2999        self.inner
3000            .watch_registry
3001            .lock()
3002            .map(|registry| registry.has_controlled_task(task_id))
3003            .unwrap_or(false)
3004    }
3005
3006    fn clear_task_watch_state(&self, task_id: &str) {
3007        if let Ok(mut registry) = self.inner.watch_registry.lock() {
3008            registry.clear_task(task_id);
3009        }
3010    }
3011
3012    pub(crate) fn scan_task_watch_output(&self, task: &Arc<BgTask>) {
3013        let mode = match task.state.lock() {
3014            Ok(state) => state.metadata.mode.clone(),
3015            Err(_) => return,
3016        };
3017        let mut stdout = (mode == BgMode::Pipes)
3018            .then(|| open_task_artifact(&task.paths, TaskArtifact::Stdout))
3019            .transpose()
3020            .ok()
3021            .flatten();
3022        let mut stderr = (mode == BgMode::Pipes)
3023            .then(|| open_task_artifact(&task.paths, TaskArtifact::Stderr))
3024            .transpose()
3025            .ok()
3026            .flatten();
3027        let mut pty = (mode == BgMode::Pty)
3028            .then(|| open_task_artifact(&task.paths, TaskArtifact::Pty))
3029            .transpose()
3030            .ok()
3031            .flatten();
3032        let mut matches = Vec::new();
3033        if let Ok(mut registry) = self.inner.watch_registry.lock() {
3034            match mode {
3035                BgMode::Pipes => {
3036                    let (Some(stdout), Some(stderr)) = (stdout.as_mut(), stderr.as_mut()) else {
3037                        return;
3038                    };
3039                    let stdout_key = format!("{}:stdout", task.task_id);
3040                    let stderr_key = format!("{}:stderr", task.task_id);
3041                    matches.extend(registry.scan_file_new_bytes(
3042                        &stdout_key,
3043                        &task.task_id,
3044                        stdout,
3045                    ));
3046                    matches.extend(registry.scan_file_new_bytes(
3047                        &stderr_key,
3048                        &task.task_id,
3049                        stderr,
3050                    ));
3051                }
3052                BgMode::Pty => {
3053                    let Some(pty) = pty.as_mut() else {
3054                        return;
3055                    };
3056                    let pty_key = format!("{}:pty", task.task_id);
3057                    matches.extend(registry.scan_file_new_bytes(&pty_key, &task.task_id, pty));
3058                }
3059            }
3060        }
3061        let (stdout_offset, stderr_offset, pty_offset) = self.watch_stream_cursors(&task.task_id);
3062        if matches.is_empty() {
3063            // Advance durable cursors even when nothing matched so a restart
3064            // does not re-scan already-observed bytes.
3065            if self.task_has_watch_control(&task.task_id) {
3066                self.persist_task_watch_cursors(
3067                    &task.session_id,
3068                    &task.task_id,
3069                    stdout_offset,
3070                    stderr_offset,
3071                    pty_offset,
3072                );
3073            }
3074            return;
3075        }
3076        for pattern_match in matches {
3077            self.persist_watch_match(
3078                &task.session_id,
3079                &task.task_id,
3080                &pattern_match,
3081                stdout_offset,
3082                stderr_offset,
3083                pty_offset,
3084            );
3085            self.emit_bash_pattern_match(&task.session_id, pattern_match);
3086        }
3087        self.persist_task_watch_cursors(
3088            &task.session_id,
3089            &task.task_id,
3090            stdout_offset,
3091            stderr_offset,
3092            pty_offset,
3093        );
3094    }
3095
3096    pub(crate) fn observed_status(
3097        &self,
3098        task_id: &str,
3099        session_id: &str,
3100        preview_bytes: usize,
3101    ) -> Option<BgTaskSnapshot> {
3102        validate_task_id(task_id).ok()?;
3103        let task = self.task_for_session(task_id, session_id)?;
3104        Some(self.snapshot_with_terminal_cache(&task, preview_bytes))
3105    }
3106
3107    pub fn status(
3108        &self,
3109        task_id: &str,
3110        session_id: &str,
3111        project_root: Option<&Path>,
3112        storage_dir: Option<&Path>,
3113        preview_bytes: usize,
3114    ) -> Option<BgTaskSnapshot> {
3115        validate_task_id(task_id).ok()?;
3116        let terminal_db_fallback_allowed = storage_dir
3117            .is_some_and(|storage_dir| task_bundle_is_absent(storage_dir, session_id, task_id));
3118        let mut task = self.task_for_session(task_id, session_id);
3119        if task.is_none() {
3120            if let Some(storage_dir) = storage_dir {
3121                let _ = if let Some(project_root) = project_root {
3122                    self.replay_session_for_project(storage_dir, session_id, project_root)
3123                } else {
3124                    self.replay_session(storage_dir, session_id)
3125                };
3126                task = self.task_for_session(task_id, session_id);
3127            }
3128        }
3129        let Some(task) = task else {
3130            if terminal_db_fallback_allowed {
3131                if let Some(snapshot) = storage_dir.and_then(|storage_dir| {
3132                    self.terminal_db_status_for_session(session_id, task_id, storage_dir)
3133                }) {
3134                    return Some(snapshot);
3135                }
3136            }
3137            return self.status_relaxed(
3138                task_id,
3139                session_id,
3140                project_root?,
3141                storage_dir?,
3142                preview_bytes,
3143                terminal_db_fallback_allowed,
3144            );
3145        };
3146        let _ = self.poll_task(&task);
3147        Some(self.snapshot_with_terminal_cache(&task, preview_bytes))
3148    }
3149
3150    fn status_relaxed_task(
3151        &self,
3152        task_id: &str,
3153        project_root: &Path,
3154        storage_dir: &Path,
3155    ) -> Option<Arc<BgTask>> {
3156        validate_task_id(task_id).ok()?;
3157        let canonical_project = canonicalized_path(project_root);
3158        match self.lookup_relaxed_task_from_db(task_id, project_root) {
3159            Some(Ok(Some(row))) => {
3160                let metadata = PersistedTask::from(row);
3161                if let Some(task) = self.task(task_id) {
3162                    let matches_project = task
3163                        .state
3164                        .lock()
3165                        .map(|state| {
3166                            state
3167                                .metadata
3168                                .project_root
3169                                .as_deref()
3170                                .map(canonicalized_path)
3171                                .as_deref()
3172                                == Some(canonical_project.as_path())
3173                        })
3174                        .unwrap_or(false);
3175                    return matches_project.then_some(task);
3176                }
3177                let resolved = resolve_task_layout(
3178                    &session_tasks_dir(storage_dir, &metadata.session_id),
3179                    &metadata.task_id,
3180                )
3181                .ok()?;
3182                let disk = read_task_at(&resolved).ok()?;
3183                if disk.task_id != metadata.task_id || disk.session_id != metadata.session_id {
3184                    return None;
3185                }
3186                if self
3187                    .insert_rehydrated_task(metadata, resolved.paths, true)
3188                    .is_err()
3189                {
3190                    return None;
3191                }
3192                return self.task(task_id);
3193            }
3194            Some(Ok(None)) => {
3195                crate::slog_info!(
3196                    "bash task relaxed DB miss for {}; falling back to disk",
3197                    task_id
3198                );
3199            }
3200            Some(Err(error)) => {
3201                crate::slog_warn!(
3202                    "bash task relaxed DB lookup failed for {}; falling back to disk: {}",
3203                    task_id,
3204                    error
3205                );
3206            }
3207            None => {
3208                crate::slog_info!(
3209                    "bash task relaxed DB unavailable for {}; falling back to disk",
3210                    task_id
3211                );
3212            }
3213        }
3214        let root = storage_dir.join("bash-tasks");
3215        let entries = fs::read_dir(&root).ok()?;
3216        for entry in entries.flatten() {
3217            let dir = entry.path();
3218            if !dir.is_dir() {
3219                continue;
3220            }
3221            let resolved = match resolve_task_layout(&dir, task_id) {
3222                Ok(task) => task,
3223                Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue,
3224                Err(error) => {
3225                    if self.db_has_live_process_for_task(task_id) {
3226                        crate::slog_warn!(
3227                            "refusing to quarantine unresolved live background task {task_id} during relaxed lookup: {error}"
3228                        );
3229                        continue;
3230                    }
3231                    crate::slog_warn!(
3232                        "quarantining unresolved background task {task_id} during relaxed lookup: {error}"
3233                    );
3234                    let _ = quarantine_task_layout(storage_dir, &dir, task_id, "invalid");
3235                    continue;
3236                }
3237            };
3238            let metadata = match read_task_at(&resolved) {
3239                Ok(metadata) => metadata,
3240                Err(error) => {
3241                    if self.db_has_live_process_for_task(task_id) {
3242                        crate::slog_warn!(
3243                            "refusing to quarantine unreadable live background task {task_id} during relaxed lookup: {error}"
3244                        );
3245                        continue;
3246                    }
3247                    crate::slog_warn!(
3248                        "quarantining invalid background task metadata {task_id} during relaxed lookup: {error}"
3249                    );
3250                    let _ = quarantine_task_layout(storage_dir, &dir, task_id, "invalid");
3251                    continue;
3252                }
3253            };
3254            let metadata_project = metadata.project_root.as_deref().map(canonicalized_path);
3255            if metadata_project.as_deref() != Some(canonical_project.as_path()) {
3256                continue;
3257            }
3258            if let Some(task) = self.task(task_id) {
3259                let matches_project = task
3260                    .state
3261                    .lock()
3262                    .map(|state| {
3263                        state
3264                            .metadata
3265                            .project_root
3266                            .as_deref()
3267                            .map(canonicalized_path)
3268                            .as_deref()
3269                            == Some(canonical_project.as_path())
3270                    })
3271                    .unwrap_or(false);
3272                return matches_project.then_some(task);
3273            }
3274            if self
3275                .insert_rehydrated_task(metadata, resolved.paths, true)
3276                .is_err()
3277            {
3278                return None;
3279            }
3280            return self.task(task_id);
3281        }
3282        None
3283    }
3284
3285    fn lookup_relaxed_task_from_db(
3286        &self,
3287        task_id: &str,
3288        project_root: &Path,
3289    ) -> Option<Result<Option<BashTaskRow>, String>> {
3290        let pool = self
3291            .inner
3292            .db_pool
3293            .read()
3294            .ok()
3295            .and_then(|slot| slot.clone())?;
3296        let harness = self
3297            .inner
3298            .db_harness
3299            .read()
3300            .ok()
3301            .and_then(|slot| slot.clone())?;
3302        let conn = match pool.lock() {
3303            Ok(conn) => conn,
3304            Err(_) => return Some(Err("db mutex poisoned".to_string())),
3305        };
3306        let project_key = crate::path_identity::project_scope_key(project_root);
3307        Some(
3308            crate::db::bash_tasks::find_bash_task_for_project(
3309                &conn,
3310                &harness,
3311                &project_key,
3312                task_id,
3313            )
3314            .map_err(|error| error.to_string()),
3315        )
3316    }
3317
3318    pub(super) fn status_relaxed(
3319        &self,
3320        task_id: &str,
3321        _session_id: &str,
3322        project_root: &Path,
3323        storage_dir: &Path,
3324        preview_bytes: usize,
3325        allow_terminal_db_fallback: bool,
3326    ) -> Option<BgTaskSnapshot> {
3327        let fallback_row = if allow_terminal_db_fallback {
3328            self.lookup_relaxed_task_from_db(task_id, project_root)
3329        } else {
3330            None
3331        }
3332        .and_then(Result::ok)
3333        .flatten()
3334        .filter(|row| task_bundle_is_absent(storage_dir, &row.session_id, &row.task_id));
3335        if let Some(task) = self.status_relaxed_task(task_id, project_root, storage_dir) {
3336            let _ = self.poll_task(&task);
3337            return Some(self.snapshot_with_terminal_cache(&task, preview_bytes));
3338        }
3339        let row = fallback_row?;
3340        let metadata = PersistedTask::from(row.clone());
3341        metadata
3342            .status
3343            .is_terminal()
3344            .then(|| terminal_db_row_snapshot(row, metadata))
3345    }
3346
3347    pub fn kill_relaxed(
3348        &self,
3349        task_id: &str,
3350        project_root: &Path,
3351        storage_dir: &Path,
3352    ) -> Result<BgTaskSnapshot, String> {
3353        let task = self
3354            .status_relaxed_task(task_id, project_root, storage_dir)
3355            .ok_or_else(|| format!("background task not found: {task_id}"))?;
3356        self.kill_with_status(task_id, &task.session_id, BgTaskStatus::Killed)
3357    }
3358
3359    pub fn maybe_gc_persisted(&self, storage_dir: &Path) -> Result<usize, String> {
3360        #[cfg(test)]
3361        self.inner.persisted_gc_runs.fetch_add(1, Ordering::SeqCst);
3362        // Recorded when the run ENDS (on every exit path, via the guard), so an
3363        // observer that sees the thread name knows the sweep is over. Replay
3364        // spawns this detached; a test that plants storage after replay returns
3365        // must wait for this before its fixture is safe from the sweep.
3366        struct RecordThreadOnExit<'a>(&'a RegistryInner);
3367        impl Drop for RecordThreadOnExit<'_> {
3368            fn drop(&mut self) {
3369                *self
3370                    .0
3371                    .persisted_gc_thread
3372                    .lock()
3373                    .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(
3374                    std::thread::current()
3375                        .name()
3376                        .unwrap_or("<unnamed>")
3377                        .to_string(),
3378                );
3379            }
3380        }
3381        let _record_thread = RecordThreadOnExit(&self.inner);
3382
3383        let mut deleted = 0usize;
3384
3385        let root = storage_dir.join("bash-tasks");
3386        if root.exists() {
3387            let session_dirs = fs::read_dir(&root).map_err(|e| {
3388                format!(
3389                    "failed to read background task root {}: {e}",
3390                    root.display()
3391                )
3392            })?;
3393            for session_entry in session_dirs.flatten() {
3394                let session_dir = session_entry.path();
3395                if !session_dir.is_dir() {
3396                    continue;
3397                }
3398                // Replay recovery rewrites the metadata of every task it
3399                // restores, so while it runs this directory's contents are
3400                // mid-flight rather than abandoned. Judging a task from a
3401                // half-written session is how a sweep ends up destroying a row
3402                // the recovery just put back, so skip the session entirely and
3403                // leave it to a later sweep.
3404                let recovery_before = self.session_recovery_state(&session_dir);
3405                if recovery_before.in_flight > 0 {
3406                    log::debug!(
3407                        "skipping background task session {} during GC: replay recovery in flight",
3408                        session_dir.display()
3409                    );
3410                    continue;
3411                }
3412                // Nothing below holds the session still, so every destructive
3413                // step re-reads this: a recovery that starts and finishes while
3414                // the sweep is making up its mind about a task leaves the
3415                // sweep's evidence stale, and stale evidence must not destroy.
3416                let undisturbed = |registry: &Self| {
3417                    registry.session_recovery_state(&session_dir) == recovery_before
3418                };
3419                let (task_ids, invalid_entries) = match discover_task_ids(&session_dir) {
3420                    Ok(discovery) => discovery,
3421                    Err(error) => {
3422                        crate::slog_warn!(
3423                            "failed to discover background task session {}: {error}",
3424                            session_dir.display()
3425                        );
3426                        continue;
3427                    }
3428                };
3429                for entry in invalid_entries {
3430                    if !undisturbed(self) {
3431                        break;
3432                    }
3433                    let _ = quarantine_invalid_entry(storage_dir, &session_dir, &entry);
3434                }
3435                for task_id in task_ids {
3436                    let resolved = match resolve_task_layout(&session_dir, &task_id) {
3437                        Ok(task) => task,
3438                        // Uncertainty never quarantines: if the age probe itself
3439                        // fails (the directory vanished or is mid-creation), skip
3440                        // this pass; a genuinely abandoned layout is still there
3441                        // for the next one.
3442                        Err(error)
3443                            if error.kind() == std::io::ErrorKind::NotFound
3444                                && uninitialized_layout_is_recent(
3445                                    &session_dir,
3446                                    &task_id,
3447                                    Duration::from_secs(5 * 60),
3448                                )
3449                                .unwrap_or(true) =>
3450                        {
3451                            continue;
3452                        }
3453                        Err(error) => {
3454                            // A metadata file replaced under the open, eight times
3455                            // in a row, is a writer that never paused — proof the
3456                            // task is alive, not a layout to quarantine. Skip this
3457                            // pass; the next one reads whatever the writer left.
3458                            if concurrently_replaced(&error) {
3459                                continue;
3460                            }
3461                            if self.db_has_live_process_for_task(&task_id) {
3462                                crate::slog_warn!(
3463                                    "refusing to quarantine unresolved live background task {task_id} during GC: {error}"
3464                                );
3465                                continue;
3466                            }
3467                            if !undisturbed(self) {
3468                                continue;
3469                            }
3470                            crate::slog_warn!(
3471                                "quarantining unresolved background task {task_id}: {error}"
3472                            );
3473                            quarantine_task_layout(storage_dir, &session_dir, &task_id, "invalid")
3474                                .map_err(|error| error.to_string())?;
3475                            continue;
3476                        }
3477                    };
3478                    if modified_within(&resolved.paths.json, PERSISTED_GC_GRACE) {
3479                        continue;
3480                    }
3481                    let metadata = match read_task_at(&resolved) {
3482                        Ok(metadata) => metadata,
3483                        Err(error) => {
3484                            if concurrently_replaced(&error) {
3485                                continue;
3486                            }
3487                            if self.db_has_live_process_for_task(&task_id) {
3488                                crate::slog_warn!(
3489                                    "refusing to quarantine unreadable live background task {task_id} during GC: {error}"
3490                                );
3491                                continue;
3492                            }
3493                            if !undisturbed(self) {
3494                                continue;
3495                            }
3496                            crate::slog_warn!(
3497                                "quarantining corrupt background task metadata {task_id}: {error}"
3498                            );
3499                            quarantine_task_layout(storage_dir, &session_dir, &task_id, "corrupt")
3500                                .map_err(|error| error.to_string())?;
3501                            continue;
3502                        }
3503                    };
3504                    if !(metadata.status.is_terminal() && metadata.completion_delivered) {
3505                        continue;
3506                    }
3507                    if Self::persisted_task_process_is_alive(&metadata)
3508                        || self.db_has_live_process_for_task(&task_id)
3509                    {
3510                        crate::slog_warn!(
3511                            "refusing to delete terminal background task bundle {task_id}: recorded process is still alive"
3512                        );
3513                        continue;
3514                    }
3515                    if !undisturbed(self) {
3516                        continue;
3517                    }
3518                    match delete_task_bundle(&resolved.paths) {
3519                        Ok(()) => {
3520                            self.delete_gc_task_from_db(&metadata);
3521                            self.evaluate_erased_watch_targets();
3522                            deleted += 1;
3523                            log::debug!(
3524                                "deleted persisted background task bundle {}",
3525                                metadata.task_id
3526                            );
3527                        }
3528                        Err(error) => {
3529                            crate::slog_warn!(
3530                                "failed to delete background task bundle {}: {error}",
3531                                metadata.task_id
3532                            );
3533                        }
3534                    }
3535                }
3536            }
3537        }
3538        gc_quarantine(storage_dir);
3539        Ok(deleted)
3540    }
3541
3542    pub fn list(&self, preview_bytes: usize) -> Vec<BgTaskSnapshot> {
3543        let tasks = self
3544            .inner
3545            .tasks
3546            .lock()
3547            .map(|tasks| tasks.values().cloned().collect::<Vec<_>>())
3548            .unwrap_or_default();
3549        tasks
3550            .into_iter()
3551            .map(|task| {
3552                let _ = self.poll_task(&task);
3553                self.snapshot_with_terminal_cache(&task, preview_bytes)
3554            })
3555            .collect()
3556    }
3557
3558    /// Replace terminal pipe snapshots with the task's cached rendered output.
3559    /// Running tasks stay raw (tail-only) so agents debugging a live process see
3560    /// exactly what it emitted. PTY tasks are explicitly excluded: their raw
3561    /// terminal bytes are rendered by the plugin's PTY path, not the line
3562    /// compressor.
3563    fn maybe_compress_snapshot(&self, task: &Arc<BgTask>, snapshot: &mut BgTaskSnapshot) {
3564        if !snapshot.info.status.is_terminal() || snapshot.info.mode == BgMode::Pty {
3565            return;
3566        }
3567        if let Some(cache) = self.ensure_terminal_output_cache(task) {
3568            let mut output_preview = cache.output_preview.clone();
3569            let envelope = append_bash_output_envelope(&cache, &mut output_preview);
3570            snapshot.output_preview = output_preview;
3571            snapshot.bash_output_list_envelope = envelope;
3572            snapshot.output_truncated = cache.output_truncated;
3573        }
3574    }
3575
3576    pub fn kill(&self, task_id: &str, session_id: &str) -> Result<BgTaskSnapshot, String> {
3577        self.kill_with_status(task_id, session_id, BgTaskStatus::Killed)
3578    }
3579
3580    /// Terminate live tasks whose project root has been confirmed absent by
3581    /// subc's consecutive directory-absence scans, so tasks cannot keep using
3582    /// a root that has been verified for reclamation.
3583    ///
3584    /// The absence signal intentionally cannot distinguish deletion from
3585    /// renaming: a task's cwd handle can follow a moved directory even while
3586    /// the registered path is absent. A renamed root is nevertheless a retired
3587    /// registry identity, so killing those tasks is accepted; operational
3588    /// guidance is to rename a project only after its live tasks have ended.
3589    pub fn kill_running_tasks_for_root(&self, project_root: &Path) -> usize {
3590        let canonical_root = canonicalized_path(project_root);
3591        let targets = self
3592            .inner
3593            .tasks
3594            .lock()
3595            .map(|tasks| {
3596                tasks
3597                    .values()
3598                    .filter_map(|task| {
3599                        let state = task.state.lock().ok()?;
3600                        let status = &state.metadata.status;
3601                        let running = matches!(status, BgTaskStatus::Running)
3602                            || (state.metadata.mode == BgMode::Pty
3603                                && matches!(status, BgTaskStatus::Killing));
3604                        if !running {
3605                            return None;
3606                        }
3607                        let task_root = state
3608                            .metadata
3609                            .project_root
3610                            .as_deref()
3611                            .unwrap_or(&state.metadata.workdir);
3612                        (canonicalized_path(task_root) == canonical_root)
3613                            .then(|| (task.task_id.clone(), task.session_id.clone()))
3614                    })
3615                    .collect::<Vec<_>>()
3616            })
3617            .unwrap_or_default();
3618
3619        let mut killed = 0;
3620        for (task_id, session_id) in targets {
3621            match self.kill_with_status_reason(
3622                &task_id,
3623                &session_id,
3624                BgTaskStatus::Killed,
3625                Some(ROOT_RECLAIMED_REASON.to_string()),
3626            ) {
3627                Ok(_) => killed += 1,
3628                Err(error) => crate::slog_warn!(
3629                    "failed to terminate background task {task_id} for reclaimed root {}: {error}",
3630                    project_root.display()
3631                ),
3632            }
3633        }
3634        killed
3635    }
3636
3637    pub fn promote(&self, task_id: &str, session_id: &str) -> Result<bool, String> {
3638        let task = self
3639            .task_for_session(task_id, session_id)
3640            .ok_or_else(|| format!("background task not found: {task_id}"))?;
3641        let terminal_after_promote = {
3642            let mut state = task
3643                .state
3644                .lock()
3645                .map_err(|_| "background task lock poisoned".to_string())?;
3646            let updated = self
3647                .update_task_metadata(&task.paths, |metadata| {
3648                    metadata.notify_on_completion = true;
3649                    metadata.completion_delivered = false;
3650                })
3651                .map_err(|e| format!("failed to promote background task: {e}"))?;
3652            state.metadata = updated;
3653            state.metadata.status.is_terminal()
3654        };
3655        if terminal_after_promote {
3656            self.post_terminal_transition(&task, true)?;
3657        }
3658        Ok(true)
3659    }
3660
3661    pub(crate) fn kill_for_timeout(&self, task_id: &str, session_id: &str) -> Result<(), String> {
3662        self.kill_with_status(task_id, session_id, BgTaskStatus::TimedOut)
3663            .map(|_| ())
3664    }
3665
3666    pub fn cleanup_finished(&self, older_than: Duration) {
3667        let cutoff = Instant::now().checked_sub(older_than);
3668        let removable_paths: Vec<(String, TaskPaths)> =
3669            if let Ok(mut tasks) = self.inner.tasks.lock() {
3670                let removable = tasks
3671                    .iter()
3672                    .filter_map(|(task_id, task)| {
3673                        let delivered_terminal = task
3674                            .state
3675                            .lock()
3676                            .map(|state| {
3677                                state.metadata.status.is_terminal()
3678                                    && state.metadata.completion_delivered
3679                            })
3680                            .unwrap_or(false);
3681                        if !delivered_terminal {
3682                            return None;
3683                        }
3684
3685                        let terminal_at = task.terminal_at.lock().ok().and_then(|at| *at);
3686                        let expired = match (terminal_at, cutoff) {
3687                            (Some(terminal_at), Some(cutoff)) => terminal_at <= cutoff,
3688                            (Some(_), None) => true,
3689                            (None, _) => false,
3690                        };
3691                        expired.then(|| task_id.clone())
3692                    })
3693                    .collect::<Vec<_>>();
3694
3695                removable
3696                    .into_iter()
3697                    .filter_map(|task_id| {
3698                        tasks
3699                            .remove(&task_id)
3700                            .map(|task| (task_id, task.paths.clone()))
3701                    })
3702                    .collect()
3703            } else {
3704                Vec::new()
3705            };
3706
3707        for (task_id, paths) in removable_paths {
3708            match delete_task_bundle(&paths) {
3709                Ok(()) => log::debug!("deleted persisted background task bundle {task_id}"),
3710                Err(error) => crate::slog_warn!(
3711                    "failed to delete persisted background task bundle {task_id}: {error}"
3712                ),
3713            }
3714        }
3715    }
3716
3717    pub fn drain_completions(&self) -> Vec<BgCompletion> {
3718        self.drain_completions_for_session(None)
3719    }
3720
3721    pub fn drain_completions_for_session(&self, session_id: Option<&str>) -> Vec<BgCompletion> {
3722        if let Some(session_id) = session_id {
3723            let pending_matches = self.pending_pattern_matches_for_session(session_id);
3724            if let Ok(sender) = self
3725                .inner
3726                .progress_sender
3727                .lock()
3728                .map(|sender| sender.clone())
3729            {
3730                if let Some(sender) = sender.as_ref() {
3731                    for pattern_match in pending_matches {
3732                        sender(PushFrame::BashPatternMatch(pattern_match));
3733                    }
3734                }
3735            }
3736        }
3737        let completions = match self.inner.completions.lock() {
3738            Ok(completions) => completions,
3739            Err(_) => return Vec::new(),
3740        };
3741
3742        completions
3743            .iter()
3744            .filter(|completion| completion_matches_session(completion, session_id))
3745            .cloned()
3746            .collect()
3747    }
3748
3749    pub fn has_completions_for_session(&self, session_id: Option<&str>) -> bool {
3750        match self.inner.completions.lock() {
3751            Ok(completions) => completions
3752                .iter()
3753                .any(|completion| completion_matches_session(completion, session_id)),
3754            // Bias to safety: if the queue state cannot be inspected cheaply,
3755            // let callers take the existing drain path rather than risk
3756            // suppressing a pending completion.
3757            Err(_) => true,
3758        }
3759    }
3760
3761    pub fn unacked_wake_keys(&self) -> HashSet<String> {
3762        let mut keys = self
3763            .inner
3764            .completions
3765            .lock()
3766            .map(|completions| {
3767                completions
3768                    .iter()
3769                    .map(|completion| {
3770                        format!(
3771                            "completion\0{}\0{}",
3772                            completion.session_id, completion.task_id
3773                        )
3774                    })
3775                    .collect::<HashSet<_>>()
3776            })
3777            .unwrap_or_default();
3778        let Some((harness, pool)) = self.db_harness_and_pool() else {
3779            return keys;
3780        };
3781        let Ok(conn) = pool.lock() else {
3782            return keys;
3783        };
3784        if let Ok(rows) = crate::db::bash_watches::list_bash_pattern_watches(&conn, &harness) {
3785            keys.extend(rows.into_iter().filter(|row| row.pending_match).map(|row| {
3786                format!(
3787                    "match\0{}\0{}\0{}",
3788                    row.session_id, row.task_id, row.watch_id
3789                )
3790            }));
3791        }
3792        keys
3793    }
3794
3795    pub fn unacked_wake_count_for_session(&self, session_id: Option<&str>) -> usize {
3796        let completion_count = self
3797            .inner
3798            .completions
3799            .lock()
3800            .map(|completions| {
3801                completions
3802                    .iter()
3803                    .filter(|completion| completion_matches_session(completion, session_id))
3804                    .count()
3805            })
3806            .unwrap_or(1);
3807        let Some((harness, pool)) = self.db_harness_and_pool() else {
3808            return completion_count;
3809        };
3810        let Ok(conn) = pool.lock() else {
3811            return completion_count.saturating_add(1);
3812        };
3813        let pending_matches = match session_id {
3814            Some(session_id) => {
3815                crate::db::bash_watches::count_pending_bash_pattern_watches_for_session(
3816                    &conn, &harness, session_id,
3817                )
3818            }
3819            None => crate::db::bash_watches::count_pending_bash_pattern_watches(&conn, &harness),
3820        };
3821        completion_count.saturating_add(pending_matches.unwrap_or(1))
3822    }
3823
3824    pub fn has_unacked_wakes_for_session(&self, session_id: &str) -> bool {
3825        self.unacked_wake_count_for_session(Some(session_id)) > 0
3826    }
3827
3828    pub fn stuck_pending_watches_for_session(
3829        &self,
3830        session_id: &str,
3831        older_than: Duration,
3832    ) -> Vec<(String, String, u64)> {
3833        let Some((harness, pool)) = self.db_harness_and_pool() else {
3834            return Vec::new();
3835        };
3836        let Ok(conn) = pool.lock() else {
3837            return Vec::new();
3838        };
3839        let now = unix_millis();
3840        let minimum_age_ms = older_than.as_millis().min(u128::from(u64::MAX)) as u64;
3841        crate::db::bash_watches::list_bash_pattern_watches_for_session(&conn, &harness, session_id)
3842            .unwrap_or_default()
3843            .into_iter()
3844            .filter(|row| row.pending_match && self.task(&row.task_id).is_some())
3845            .filter_map(|row| {
3846                let created_at = u64::try_from(row.created_at).ok()?;
3847                let age_ms = now.saturating_sub(created_at);
3848                (age_ms >= minimum_age_ms).then_some((row.task_id, row.watch_id, age_ms))
3849            })
3850            .collect()
3851    }
3852
3853    pub fn ack_completions_for_session(
3854        &self,
3855        session_id: Option<&str>,
3856        task_ids: &[String],
3857    ) -> Vec<String> {
3858        if task_ids.is_empty() {
3859            return Vec::new();
3860        }
3861        let requested_task_ids = task_ids.iter().map(String::as_str).collect::<HashSet<_>>();
3862        let mut completion_sessions = HashMap::new();
3863        if let Ok(mut completions) = self.inner.completions.lock() {
3864            completions.retain(|completion| {
3865                let session_matches = session_id
3866                    .map(|session_id| completion.session_id == session_id)
3867                    .unwrap_or(true);
3868                if session_matches && requested_task_ids.contains(completion.task_id.as_str()) {
3869                    completion_sessions
3870                        .insert(completion.task_id.clone(), completion.session_id.clone());
3871                    false
3872                } else {
3873                    true
3874                }
3875            });
3876        }
3877
3878        let mut delivered = Vec::new();
3879        for task_id in task_ids {
3880            if self.has_erased_watch_reference(task_id) {
3881                if let Some((harness, pool)) = self.db_harness_and_pool() {
3882                    if let Ok(conn) = pool.lock() {
3883                        if let Ok(rows) =
3884                            crate::db::bash_watches::list_bash_pattern_watches_by_task_id(
3885                                &conn, &harness, task_id,
3886                            )
3887                        {
3888                            for row in rows {
3889                                let _ = crate::db::bash_watches::delete_bash_pattern_watch(
3890                                    &conn,
3891                                    &harness,
3892                                    &row.session_id,
3893                                    task_id,
3894                                    &row.watch_id,
3895                                );
3896                            }
3897                        }
3898                    }
3899                }
3900                self.clear_task_watch_state(task_id);
3901                // Retain the process-local erased marker so later status calls still
3902                // distinguish this task from an ID that never existed.
3903                delivered.push(task_id.clone());
3904                continue;
3905            }
3906            let task = if let Some(session_id) = session_id {
3907                self.task_for_session(task_id, session_id).or_else(|| {
3908                    completion_sessions
3909                        .contains_key(task_id)
3910                        .then(|| self.task(task_id))
3911                        .flatten()
3912                })
3913            } else if let Some(completion_session_id) = completion_sessions.get(task_id) {
3914                self.task_for_session(task_id, completion_session_id)
3915                    .or_else(|| self.task(task_id))
3916            } else {
3917                self.task(task_id)
3918            };
3919            if let Some(task) = task {
3920                let terminal = task
3921                    .state
3922                    .lock()
3923                    .map(|state| state.metadata.status.is_terminal())
3924                    .unwrap_or(false);
3925                // Pattern-watch delivery shares the completion ack lane: once the
3926                // plugin confirms the agent saw the notification, drop once-watches
3927                // (and all watches on terminal tasks) so restart cannot re-fire them.
3928                self.ack_persisted_watches_for_task(&task.session_id, task_id, terminal);
3929                if terminal {
3930                    self.clear_task_watch_state(task_id);
3931                    if task.set_completion_delivered(true, self).is_ok() {
3932                        delivered.push(task_id.clone());
3933                    }
3934                } else {
3935                    // Mid-run pattern-match ack must not flip completion_delivered —
3936                    // the task is still running and will need a real completion later.
3937                    self.sync_memory_watches_from_persistence(task_id);
3938                    delivered.push(task_id.clone());
3939                }
3940            } else if let Some(session_id) = session_id {
3941                // Another root actor can receive the session's ack while the task is
3942                // still owned by its original registry. Consult the shared row rather
3943                // than treating absence from this process-local map as terminal.
3944                let terminal = self
3945                    .persisted_task_is_terminal(session_id, task_id)
3946                    .unwrap_or(true);
3947                self.ack_persisted_watches_for_task(session_id, task_id, terminal);
3948                if terminal {
3949                    self.mark_persisted_completion_delivered(session_id, task_id);
3950                }
3951                delivered.push(task_id.clone());
3952            }
3953        }
3954
3955        delivered
3956    }
3957
3958    fn persisted_task_is_terminal(&self, session_id: &str, task_id: &str) -> Option<bool> {
3959        let (harness, pool) = self.db_harness_and_pool()?;
3960        let conn = pool.lock().ok()?;
3961        let row = crate::db::bash_tasks::get_bash_task(&conn, &harness, session_id, task_id)
3962            .ok()
3963            .flatten()?;
3964        Some(matches!(
3965            row.status.as_str(),
3966            "completed" | "failed" | "killed" | "timed_out" | "fate_unknown"
3967        ))
3968    }
3969
3970    fn mark_persisted_completion_delivered(&self, session_id: &str, task_id: &str) {
3971        let Some((harness, pool)) = self.db_harness_and_pool() else {
3972            return;
3973        };
3974        let Ok(conn) = pool.lock() else {
3975            return;
3976        };
3977        let _ = conn.execute(
3978            "UPDATE bash_tasks SET completion_delivered = 1
3979             WHERE harness = ?1 AND session_id = ?2 AND task_id = ?3
3980               AND status IN ('completed', 'failed', 'killed', 'timed_out', 'fate_unknown')",
3981            rusqlite::params![harness, session_id, task_id],
3982        );
3983    }
3984
3985    fn sync_memory_watches_from_persistence(&self, task_id: &str) {
3986        let Some((harness, pool)) = self.db_harness_and_pool() else {
3987            return;
3988        };
3989        let session_id = match self.task(task_id) {
3990            Some(task) => task.session_id.clone(),
3991            None => return,
3992        };
3993        let Ok(conn) = pool.lock() else {
3994            return;
3995        };
3996        let Ok(rows) = crate::db::bash_watches::list_bash_pattern_watches_for_task(
3997            &conn,
3998            &harness,
3999            &session_id,
4000            task_id,
4001        ) else {
4002            return;
4003        };
4004        let has_pending_match = rows.iter().any(|row| row.pending_match);
4005        let remaining: HashSet<String> = rows.into_iter().map(|row| row.watch_id).collect();
4006        if let Ok(mut registry) = self.inner.watch_registry.lock() {
4007            registry.reconcile_watch_ids(task_id, &remaining, has_pending_match);
4008        }
4009    }
4010
4011    pub fn pending_completions_for_session(&self, session_id: &str) -> Vec<BgCompletion> {
4012        self.inner
4013            .completions
4014            .lock()
4015            .map(|completions| {
4016                completions
4017                    .iter()
4018                    .filter(|completion| completion.session_id == session_id)
4019                    .cloned()
4020                    .collect()
4021            })
4022            .unwrap_or_default()
4023    }
4024
4025    fn remove_pending_completion(&self, task_id: &str) -> Option<BgCompletion> {
4026        let mut completions = self.inner.completions.lock().ok()?;
4027        let idx = completions
4028            .iter()
4029            .position(|completion| completion.task_id == task_id)?;
4030        completions.remove(idx)
4031    }
4032
4033    fn completion_snapshot_for_task(&self, task: &Arc<BgTask>) -> Option<BgCompletion> {
4034        let snapshot = self.snapshot_with_terminal_cache(task, RUNNING_OUTPUT_PREVIEW_BYTES);
4035        if !snapshot.info.status.is_terminal() {
4036            return None;
4037        }
4038        let (mut output_preview, output_truncated, cache) = if snapshot.info.mode == BgMode::Pty {
4039            (String::new(), false, None)
4040        } else {
4041            self.ensure_terminal_output_cache(task)
4042                .map(|cache| {
4043                    let (preview, truncated) =
4044                        completion_preview_for_cache(&cache, snapshot.exit_code);
4045                    (preview, truncated, Some(cache))
4046                })
4047                .unwrap_or_else(|| (String::new(), false, None))
4048        };
4049        let bash_output_list_envelope = cache
4050            .as_ref()
4051            .and_then(|cache| append_bash_output_envelope(cache, &mut output_preview));
4052        Some(BgCompletion {
4053            task_id: snapshot.info.task_id,
4054            session_id: task.session_id.clone(),
4055            status: snapshot.info.status,
4056            exit_code: snapshot.exit_code,
4057            command: snapshot.info.command,
4058            output_preview,
4059            bash_output_list_envelope,
4060            output_truncated,
4061            original_tokens: None,
4062            compressed_tokens: None,
4063            tokens_skipped: false,
4064            status_reason: snapshot.info.status_reason,
4065            live_descendants: snapshot.live_descendants,
4066            live_descendants_omitted: snapshot.live_descendants_omitted,
4067            live_descendants_summary: snapshot.live_descendants_summary,
4068        })
4069    }
4070
4071    pub fn detach(&self) {
4072        self.inner.shutdown.store(true, Ordering::SeqCst);
4073        if let Ok(mut tasks) = self.inner.tasks.lock() {
4074            for task in tasks.values() {
4075                if let Ok(mut state) = task.state.lock() {
4076                    match &mut state.runtime {
4077                        TaskRuntime::Piped(child) => *child = None,
4078                        TaskRuntime::Pty(runtime) => *runtime = None,
4079                    }
4080                    state.detached = true;
4081                }
4082            }
4083            tasks.clear();
4084        }
4085    }
4086
4087    pub fn shutdown(&self) {
4088        let tasks = self
4089            .inner
4090            .tasks
4091            .lock()
4092            .map(|tasks| {
4093                tasks
4094                    .values()
4095                    .map(|task| (task.task_id.clone(), task.session_id.clone()))
4096                    .collect::<Vec<_>>()
4097            })
4098            .unwrap_or_default();
4099        for (task_id, session_id) in tasks {
4100            let _ = self.kill(&task_id, &session_id);
4101        }
4102    }
4103
4104    pub(crate) fn poll_task(&self, task: &Arc<BgTask>) -> Result<(), String> {
4105        if let Ok(state) = task.state.lock() {
4106            if let TaskRuntime::Pty(Some(pty)) = &state.runtime {
4107                // On Windows ConPTY, the reader may not observe EOF while the
4108                // master handle is still held in `PtyRuntime`. The waiter writes
4109                // the authoritative exit marker before setting `exit_observed`,
4110                // so once exit is observed we can finalize from that marker and
4111                // drop the runtime, which lets the reader finish. Waiting for
4112                // `reader_done && exit_observed` wedges completed PTY tasks on
4113                // Windows.
4114                if !pty.exit_observed.load(Ordering::SeqCst) {
4115                    return Ok(());
4116                }
4117            }
4118        }
4119        let marker = match read_exit_marker(&task.paths) {
4120            Ok(Some(marker)) => marker,
4121            Ok(None) => return Ok(()),
4122            Err(error) => return Err(format!("failed to read exit marker: {error}")),
4123        };
4124        self.finalize_from_marker(task, marker, None)
4125    }
4126
4127    pub(crate) fn reap_child(&self, task: &Arc<BgTask>) {
4128        let mut needs_completion = false;
4129        {
4130            let Ok(mut state) = task.state.lock() else {
4131                return;
4132            };
4133            match &mut state.runtime {
4134                TaskRuntime::Piped(child_slot) => {
4135                    if let Some(child) = child_slot.as_mut() {
4136                        if let Ok(Some(status)) = child.try_wait() {
4137                            *child_slot = None;
4138                            state.detached = true;
4139                            state.child_exit_observed = true;
4140                            if let Some(handles) = state.io_handles.as_mut() {
4141                                if handles.artifact_len(TaskArtifact::Exit).unwrap_or(1) == 0 {
4142                                    let marker = status
4143                                        .code()
4144                                        .map(|code| code.to_string())
4145                                        .unwrap_or_else(|| "1".to_string());
4146                                    let _ = handles.write(TaskArtifact::Exit, marker.as_bytes());
4147                                }
4148                            }
4149                        }
4150                    } else if state.detached {
4151                        let child_known_dead = state.child_exit_observed
4152                            || state
4153                                .metadata
4154                                .child_pid
4155                                .is_some_and(|pid| !is_process_alive(pid));
4156                        if child_known_dead {
4157                            needs_completion =
4158                                self.fail_without_exit_marker_if_needed(task, &mut state);
4159                        }
4160                    }
4161                }
4162                TaskRuntime::Pty(Some(pty)) => {
4163                    if pty.exit_observed.load(Ordering::SeqCst) {
4164                        drop(state);
4165                        let _ = self.poll_task(task);
4166                        return;
4167                    }
4168                }
4169                TaskRuntime::Pty(None) => {}
4170            }
4171        }
4172        if needs_completion {
4173            let _ = self.post_terminal_transition(task, true);
4174        }
4175    }
4176
4177    fn fail_without_exit_marker_if_needed(
4178        &self,
4179        task: &Arc<BgTask>,
4180        state: &mut BgTaskState,
4181    ) -> bool {
4182        if state.metadata.status.is_terminal() {
4183            return false;
4184        }
4185        if matches!(read_exit_marker(&task.paths), Ok(Some(_))) {
4186            return false;
4187        }
4188        let child_exit_observed = state.child_exit_observed;
4189        let updated = self.update_task_metadata(&task.paths, |metadata| {
4190            let (status, reason) = if child_exit_observed {
4191                (
4192                    BgTaskStatus::Failed,
4193                    "process exited without exit marker".to_string(),
4194                )
4195            } else {
4196                (
4197                    BgTaskStatus::FateUnknown,
4198                    restart_fate_unknown_reason(metadata, &task.paths),
4199                )
4200            };
4201            metadata.mark_terminal(status, None, Some(reason));
4202        });
4203        if let Ok(metadata) = updated {
4204            state.pending_terminal_override = None;
4205            state.metadata = metadata;
4206            task.mark_terminal_now();
4207            return true;
4208        }
4209        false
4210    }
4211
4212    pub(crate) fn running_tasks(&self) -> Vec<Arc<BgTask>> {
4213        self.inner
4214            .tasks
4215            .lock()
4216            .map(|tasks| {
4217                tasks
4218                    .values()
4219                    .filter(|task| task.is_running())
4220                    .cloned()
4221                    .collect()
4222            })
4223            .unwrap_or_default()
4224    }
4225
4226    fn insert_rehydrated_task(
4227        &self,
4228        metadata: PersistedTask,
4229        paths: TaskPaths,
4230        detached: bool,
4231    ) -> Result<(), String> {
4232        let task_id = metadata.task_id.clone();
4233        let session_id = metadata.session_id.clone();
4234        let started = started_instant_from_unix_millis(metadata.started_at);
4235        let suppress_replayed_running_reminder = metadata.status == BgTaskStatus::Running;
4236        let mode = metadata.mode.clone();
4237        let task = Arc::new(BgTask {
4238            task_id: task_id.clone(),
4239            session_id,
4240            paths: paths.clone(),
4241            artifact_root: canonical_artifact_root(&paths),
4242            started,
4243            last_reminder_at: Mutex::new(suppress_replayed_running_reminder.then(Instant::now)),
4244            terminal_at: Mutex::new(metadata.status.is_terminal().then(Instant::now)),
4245            state: Mutex::new(BgTaskState {
4246                metadata,
4247                runtime: if mode == BgMode::Pty {
4248                    TaskRuntime::Pty(None)
4249                } else {
4250                    TaskRuntime::Piped(None)
4251                },
4252                io_handles: None,
4253                detached,
4254                // Replay path: we never observed the child handle's exit
4255                // in this process (the previous AFT process did, but its
4256                // observation didn't survive restart). Leave this false so
4257                // the second-pass reap falls through to the
4258                // `is_process_alive(child_pid)` probe rather than declaring
4259                // failure based on stale evidence.
4260                child_exit_observed: false,
4261                descendant_sampling_started: false,
4262                buffer: BgBuffer::registered(&paths, mode.clone()),
4263                terminal_output_cache: None,
4264                pending_terminal_override: None,
4265            }),
4266        });
4267        self.record_live_delivery_session(&task.session_id);
4268        self.inner
4269            .tasks
4270            .lock()
4271            .map_err(|_| "background task registry lock poisoned".to_string())?
4272            .insert(task_id.clone(), Arc::clone(&task));
4273        // Re-arm durable pattern watches after the task is addressable again so
4274        // gap matches (bytes written while the bridge was down) are scanned and
4275        // pending undelivered matches are re-pushed.
4276        self.rearm_persisted_watches(&task);
4277        Ok(())
4278    }
4279
4280    fn rearm_persisted_watches(&self, task: &Arc<BgTask>) {
4281        let Some((harness, pool)) = self.db_harness_and_pool() else {
4282            return;
4283        };
4284        let rows = {
4285            let Ok(conn) = pool.lock() else {
4286                return;
4287            };
4288            match crate::db::bash_watches::list_bash_pattern_watches_for_task(
4289                &conn,
4290                &harness,
4291                &task.session_id,
4292                &task.task_id,
4293            ) {
4294                Ok(rows) if !rows.is_empty() => rows,
4295                _ => return,
4296            }
4297        };
4298
4299        let mode = match task.state.lock() {
4300            Ok(state) => state.metadata.mode.clone(),
4301            Err(_) => return,
4302        };
4303        let terminal = task
4304            .state
4305            .lock()
4306            .map(|state| state.metadata.status.is_terminal())
4307            .unwrap_or(false);
4308        let completion_delivered = task
4309            .state
4310            .lock()
4311            .map(|state| state.metadata.completion_delivered)
4312            .unwrap_or(true);
4313
4314        let mut stdout = (mode == BgMode::Pipes)
4315            .then(|| open_task_artifact(&task.paths, TaskArtifact::Stdout))
4316            .transpose()
4317            .ok()
4318            .flatten();
4319        let mut stderr = (mode == BgMode::Pipes)
4320            .then(|| open_task_artifact(&task.paths, TaskArtifact::Stderr))
4321            .transpose()
4322            .ok()
4323            .flatten();
4324        let mut pty = (mode == BgMode::Pty)
4325            .then(|| open_task_artifact(&task.paths, TaskArtifact::Pty))
4326            .transpose()
4327            .ok()
4328            .flatten();
4329
4330        let mut pending_to_emit = Vec::new();
4331        let mut gap_matches = Vec::new();
4332        {
4333            let Ok(mut registry) = self.inner.watch_registry.lock() else {
4334                return;
4335            };
4336            let stdout_key = format!("{}:stdout", task.task_id);
4337            let stderr_key = format!("{}:stderr", task.task_id);
4338            let pty_key = format!("{}:pty", task.task_id);
4339
4340            // All rows for a task share stream cursors; take them from the first row.
4341            let first = &rows[0];
4342            match mode {
4343                BgMode::Pipes => {
4344                    registry.set_file_cursor(&stdout_key, first.stdout_offset.max(0) as u64);
4345                    registry.set_file_cursor(&stderr_key, first.stderr_offset.max(0) as u64);
4346                }
4347                BgMode::Pty => {
4348                    registry.set_file_cursor(&pty_key, first.pty_offset.max(0) as u64);
4349                }
4350            }
4351
4352            for row in &rows {
4353                let Ok(pattern) = WatchPattern::from_persisted(&row.pattern_kind, &row.pattern)
4354                else {
4355                    crate::slog_warn!(
4356                        "skipping unreadable persisted watch {}/{}",
4357                        row.task_id,
4358                        row.watch_id
4359                    );
4360                    continue;
4361                };
4362                if let Err(error) = registry.restore(
4363                    row.watch_id.clone(),
4364                    row.task_id.clone(),
4365                    pattern,
4366                    row.once,
4367                    row.scanning,
4368                ) {
4369                    crate::slog_warn!(
4370                        "failed to restore watch {}/{}: {error}",
4371                        row.task_id,
4372                        row.watch_id
4373                    );
4374                    continue;
4375                }
4376                if row.pending_match {
4377                    if let (Some(match_text), Some(match_offset), Some(context)) = (
4378                        row.match_text.clone(),
4379                        row.match_offset,
4380                        row.match_context.clone(),
4381                    ) {
4382                        pending_to_emit.push(PatternMatch {
4383                            watch_id: row.watch_id.clone(),
4384                            task_id: row.task_id.clone(),
4385                            match_text,
4386                            match_offset: match_offset.max(0) as u64,
4387                            context,
4388                            once: row.once,
4389                        });
4390                    }
4391                }
4392            }
4393
4394            // Gap scan: bytes written after the last persisted cursor while the
4395            // previous process was down. Skip when we already have a pending
4396            // once-match to re-deliver (avoids double-firing the same hit).
4397            let should_gap_scan =
4398                rows.iter().any(|row| row.scanning) && !pending_to_emit.iter().any(|m| m.once);
4399            if should_gap_scan {
4400                match mode {
4401                    BgMode::Pipes => {
4402                        if let (Some(stdout), Some(stderr)) = (stdout.as_mut(), stderr.as_mut()) {
4403                            gap_matches.extend(registry.scan_file_new_bytes(
4404                                &stdout_key,
4405                                &task.task_id,
4406                                stdout,
4407                            ));
4408                            gap_matches.extend(registry.scan_file_new_bytes(
4409                                &stderr_key,
4410                                &task.task_id,
4411                                stderr,
4412                            ));
4413                        }
4414                    }
4415                    BgMode::Pty => {
4416                        if let Some(pty) = pty.as_mut() {
4417                            gap_matches.extend(registry.scan_file_new_bytes(
4418                                &pty_key,
4419                                &task.task_id,
4420                                pty,
4421                            ));
4422                        }
4423                    }
4424                }
4425            }
4426        }
4427
4428        let (stdout_offset, stderr_offset, pty_offset) = self.watch_stream_cursors(&task.task_id);
4429        for pattern_match in &gap_matches {
4430            self.persist_watch_match(
4431                &task.session_id,
4432                &task.task_id,
4433                pattern_match,
4434                stdout_offset,
4435                stderr_offset,
4436                pty_offset,
4437            );
4438        }
4439        if !gap_matches.is_empty() || rows.iter().any(|row| row.scanning) {
4440            self.persist_task_watch_cursors(
4441                &task.session_id,
4442                &task.task_id,
4443                stdout_offset,
4444                stderr_offset,
4445                pty_offset,
4446            );
4447        }
4448
4449        // Prefer a single delivery: pending re-push first, else fresh gap matches.
4450        let emitted_pending = !pending_to_emit.is_empty();
4451        let to_emit = if emitted_pending {
4452            pending_to_emit
4453        } else {
4454            gap_matches
4455        };
4456        for pattern_match in to_emit {
4457            self.emit_bash_pattern_match(&task.session_id, pattern_match);
4458        }
4459
4460        if !terminal {
4461            return;
4462        }
4463
4464        let (watch_controlled, watch_matched) = self.task_watch_state(&task.task_id);
4465        if !watch_controlled {
4466            return;
4467        }
4468        if watch_matched {
4469            // The pending pattern row is the durable item, so do not also drain a
4470            // generic completion for the same terminal task.
4471            let _ = self.remove_pending_completion(&task.task_id);
4472            return;
4473        }
4474        if completion_delivered {
4475            // Already acked before restart — drop durable rows and memory state.
4476            self.clear_task_watch_state(&task.task_id);
4477            self.delete_persisted_watches_for_task(&task.session_id, &task.task_id);
4478            return;
4479        }
4480        if let Some(completion) = self
4481            .remove_pending_completion(&task.task_id)
4482            .or_else(|| self.completion_snapshot_for_task(task))
4483        {
4484            self.record_bash_watch_exit(&completion, true);
4485        }
4486        // Keep durable watches until bash_ack_completions confirms delivery.
4487        self.clear_task_watch_state(&task.task_id);
4488    }
4489
4490    fn kill_with_status(
4491        &self,
4492        task_id: &str,
4493        session_id: &str,
4494        terminal_status: BgTaskStatus,
4495    ) -> Result<BgTaskSnapshot, String> {
4496        self.kill_with_status_reason(task_id, session_id, terminal_status, None)
4497    }
4498
4499    fn kill_with_status_reason(
4500        &self,
4501        task_id: &str,
4502        session_id: &str,
4503        terminal_status: BgTaskStatus,
4504        reason: Option<String>,
4505    ) -> Result<BgTaskSnapshot, String> {
4506        let task = self
4507            .task_for_session(task_id, session_id)
4508            .ok_or_else(|| format!("background task not found: {task_id}"))?;
4509        let mut terminalized = false;
4510        #[cfg_attr(not(unix), allow(unused_mut))]
4511        let mut kill_signaled = false;
4512        #[cfg_attr(not(unix), allow(unused_mut))]
4513        let mut kill_reached = 0;
4514
4515        {
4516            let mut state = task
4517                .state
4518                .lock()
4519                .map_err(|_| "background task lock poisoned".to_string())?;
4520            if state.metadata.status.is_terminal() {
4521                state.pending_terminal_override = None;
4522                #[cfg(unix)]
4523                let live_count = state
4524                    .metadata
4525                    .live_descendants
4526                    .as_ref()
4527                    .map(Vec::len)
4528                    .unwrap_or(0)
4529                    + state.metadata.live_descendants_omitted;
4530                #[cfg(unix)]
4531                if live_count > 0 {
4532                    if let Some(pgid) = state.metadata.pgid {
4533                        kill_signaled = true;
4534                        kill_reached = live_count;
4535                        terminate_pgid(pgid, None);
4536                        let sample = live_process_group_members(pgid);
4537                        state.metadata.live_descendants =
4538                            sample.as_ref().map(|(members, _)| members.clone());
4539                        state.metadata.live_descendants_omitted =
4540                            sample.as_ref().map(|(_, omitted)| *omitted).unwrap_or(0);
4541                        self.persist_task(&task.paths, &state.metadata)
4542                            .map_err(|error| {
4543                                format!("failed to persist post-kill descendant sample: {error}")
4544                            })?;
4545                    }
4546                }
4547            } else if let Ok(Some(marker)) = read_exit_marker(&task.paths) {
4548                state.metadata =
4549                    terminal_metadata_from_marker(state.metadata.clone(), marker, reason.clone());
4550
4551                state.pending_terminal_override = None;
4552                task.mark_terminal_now();
4553                match &mut state.runtime {
4554                    // Exit marker already present: the child finished on its
4555                    // own before this kill observed it. Reap it rather than
4556                    // dropping the handle so it doesn't become a zombie
4557                    // (issue #91). The active-kill branch below already
4558                    // `wait()`s after signaling, so this is the only kill
4559                    // path that needed the explicit reap.
4560                    TaskRuntime::Piped(child_slot) => reap_piped_child(child_slot),
4561                    TaskRuntime::Pty(runtime) => *runtime = None,
4562                }
4563                state.detached = true;
4564                self.persist_task(&task.paths, &state.metadata)
4565                    .map_err(|e| format!("failed to persist terminal state: {e}"))?;
4566                terminalized = true;
4567            } else {
4568                let was_already_killing = state.metadata.status == BgTaskStatus::Killing;
4569                if !was_already_killing {
4570                    state.metadata.status = BgTaskStatus::Killing;
4571                }
4572                if reason.is_some() {
4573                    state.metadata.status_reason = reason.clone();
4574                }
4575                if !was_already_killing || reason.is_some() {
4576                    self.persist_task(&task.paths, &state.metadata)
4577                        .map_err(|e| format!("failed to persist killing state: {e}"))?;
4578                }
4579
4580                #[cfg(unix)]
4581                let pgid = state.metadata.pgid;
4582                #[cfg(windows)]
4583                let child_pid = state.metadata.child_pid;
4584                if !was_already_killing
4585                    && state.metadata.mode == BgMode::Pty
4586                    && terminal_status == BgTaskStatus::TimedOut
4587                {
4588                    state.pending_terminal_override = Some(BgTaskStatus::TimedOut);
4589                }
4590
4591                #[cfg(windows)]
4592                let mut pty_forced_terminal_status: Option<BgTaskStatus> = None;
4593
4594                match &mut state.runtime {
4595                    TaskRuntime::Piped(child_slot) => {
4596                        #[cfg(unix)]
4597                        if let Some(pgid) = pgid {
4598                            terminate_pgid(pgid, child_slot.as_mut());
4599                        }
4600                        #[cfg(windows)]
4601                        if let Some(child) = child_slot.as_mut() {
4602                            super::process::terminate_process(child);
4603                        } else if let Some(pid) = child_pid {
4604                            terminate_pid(pid);
4605                        }
4606                        if let Some(child) = child_slot.as_mut() {
4607                            let _ = child.wait();
4608                        }
4609                        *child_slot = None;
4610                        state.detached = true;
4611
4612                        if let Some(handles) = state.io_handles.as_mut() {
4613                            match handles.write(TaskArtifact::Exit, b"killed") {
4614                                Ok(()) => {}
4615                                Err(error)
4616                                    if error.kind() == std::io::ErrorKind::Interrupted
4617                                        && error.to_string().contains(
4618                                            super::persistence::ARTIFACT_CONCURRENTLY_REPLACED,
4619                                        ) =>
4620                                {
4621                                    // The child's own temp+rename exit write landed
4622                                    // between wait() and this write, leaving the
4623                                    // retained handle at zero links (Windows). The
4624                                    // replacement is the child's real exit marker;
4625                                    // re-open by path and keep whichever is there.
4626                                    write_kill_marker_if_absent(&task.paths).map_err(|e| {
4627                                        format!("failed to write kill marker after replace: {e}")
4628                                    })?;
4629                                }
4630                                Err(error) => {
4631                                    return Err(format!(
4632                                        "failed to write retained kill marker: {error}"
4633                                    ));
4634                                }
4635                            }
4636                        } else {
4637                            write_kill_marker_if_absent(&task.paths)
4638                                .map_err(|e| format!("failed to write kill marker: {e}"))?;
4639                        }
4640
4641                        let exit_code = terminal_exit_code_for_status(&terminal_status);
4642                        state
4643                            .metadata
4644                            .mark_terminal(terminal_status, exit_code, reason.clone());
4645
4646                        state.pending_terminal_override = None;
4647                        task.mark_terminal_now();
4648                        self.persist_task(&task.paths, &state.metadata)
4649                            .map_err(|e| format!("failed to persist killed state: {e}"))?;
4650                        terminalized = true;
4651                    }
4652                    TaskRuntime::Pty(Some(pty)) => {
4653                        pty.was_killed.store(true, Ordering::SeqCst);
4654                        if let Err(error) = pty.killer.kill() {
4655                            crate::slog_warn!(
4656                                "[pty-kill] {task_id} ChildKiller::kill failed: {error}"
4657                            );
4658                        }
4659                        if let Some(pid) = pty.child_pid {
4660                            #[cfg(unix)]
4661                            terminate_pgid(pid as i32, None);
4662                            #[cfg(windows)]
4663                            terminate_pid(pid);
4664                        }
4665                        drop(pty.master.take());
4666
4667                        #[cfg(windows)]
4668                        {
4669                            let default_status = if terminal_status == BgTaskStatus::TimedOut {
4670                                BgTaskStatus::TimedOut
4671                            } else {
4672                                BgTaskStatus::Killed
4673                            };
4674                            pty_forced_terminal_status = Some(
4675                                state
4676                                    .pending_terminal_override
4677                                    .take()
4678                                    .unwrap_or(default_status),
4679                            );
4680                        }
4681                    }
4682                    TaskRuntime::Pty(None) => {}
4683                }
4684
4685                #[cfg(windows)]
4686                if let Some(target_status) = pty_forced_terminal_status {
4687                    if !task.paths.exit.exists() {
4688                        write_kill_marker_if_absent(&task.paths)
4689                            .map_err(|e| format!("failed to write kill marker: {e}"))?;
4690                    }
4691
4692                    let exit_code = terminal_exit_code_for_status(&target_status);
4693                    state
4694                        .metadata
4695                        .mark_terminal(target_status, exit_code, reason.clone());
4696
4697                    state.pending_terminal_override = None;
4698                    task.mark_terminal_now();
4699                    if let TaskRuntime::Pty(runtime) = &mut state.runtime {
4700                        *runtime = None;
4701                    }
4702                    state.detached = true;
4703                    self.persist_task(&task.paths, &state.metadata)
4704                        .map_err(|e| format!("failed to persist killed PTY state: {e}"))?;
4705                    terminalized = true;
4706                }
4707            }
4708        }
4709
4710        if terminalized {
4711            self.post_terminal_transition(&task, true)?;
4712        }
4713        let mut snapshot = self.snapshot_with_terminal_cache(&task, RUNNING_OUTPUT_PREVIEW_BYTES);
4714        snapshot.kill_signaled = kill_signaled;
4715        snapshot.kill_reached = kill_reached;
4716        Ok(snapshot)
4717    }
4718
4719    fn finalize_from_marker(
4720        &self,
4721        task: &Arc<BgTask>,
4722        marker: ExitMarker,
4723        reason: Option<String>,
4724    ) -> Result<(), String> {
4725        let mut pty_reader_done = None;
4726        {
4727            let mut state = task
4728                .state
4729                .lock()
4730                .map_err(|_| "background task lock poisoned".to_string())?;
4731            if state.metadata.status.is_terminal() {
4732                state.pending_terminal_override = None;
4733                return Ok(());
4734            }
4735
4736            let pending_override = state.pending_terminal_override.take();
4737            let is_pty = state.metadata.mode == BgMode::Pty;
4738            let reason = reason.or_else(|| state.metadata.status_reason.clone());
4739            let updated = self
4740                .update_task_metadata(&task.paths, |metadata| {
4741                    let new_metadata = if is_pty && marker == ExitMarker::Killed {
4742                        let mut metadata = metadata.clone();
4743                        let target_status = pending_override.unwrap_or(BgTaskStatus::Killed);
4744                        let exit_code = terminal_exit_code_for_status(&target_status);
4745                        metadata.mark_terminal(target_status, exit_code, reason.clone());
4746                        metadata
4747                    } else {
4748                        terminal_metadata_from_marker(metadata.clone(), marker, reason.clone())
4749                    };
4750                    *metadata = new_metadata;
4751                })
4752                .map_err(|e| format!("failed to persist terminal state: {e}"))?;
4753            state.metadata = updated;
4754            task.mark_terminal_now();
4755            match &mut state.runtime {
4756                // Reap the exited direct child instead of dropping it, so it
4757                // does not linger as a `<defunct>` zombie (issue #91). The
4758                // wrapper writes the exit marker as its final act, so the
4759                // child is already exiting and `wait()` returns immediately.
4760                TaskRuntime::Piped(child_slot) => reap_piped_child(child_slot),
4761                TaskRuntime::Pty(runtime) => {
4762                    pty_reader_done = runtime
4763                        .as_ref()
4764                        .map(|runtime| Arc::clone(&runtime.reader_done));
4765                    *runtime = None;
4766                }
4767            }
4768            state.detached = true;
4769        }
4770
4771        if let Some(reader_done) = pty_reader_done {
4772            let deadline = Instant::now() + Duration::from_millis(200);
4773            while !reader_done.load(Ordering::SeqCst) && Instant::now() < deadline {
4774                std::thread::sleep(Duration::from_millis(10));
4775            }
4776        }
4777
4778        // One final scan runs before terminal notification routing so bytes
4779        // printed immediately before exit can win over the exit safety net.
4780        self.scan_task_watch_output(task);
4781
4782        self.post_terminal_transition(task, true)
4783    }
4784
4785    fn enqueue_completion_if_needed(
4786        &self,
4787        metadata: &PersistedTask,
4788        paths: Option<&TaskPaths>,
4789        emit_frame: bool,
4790    ) {
4791        if metadata.status.is_terminal() && !metadata.completion_delivered {
4792            let cache =
4793                paths.and_then(|paths| self.render_terminal_output_from_paths(metadata, paths));
4794            self.enqueue_completion_from_parts(metadata, None, paths, emit_frame, cache.as_ref());
4795        }
4796    }
4797
4798    fn render_terminal_output_from_paths(
4799        &self,
4800        metadata: &PersistedTask,
4801        paths: &TaskPaths,
4802    ) -> Option<TerminalOutputCache> {
4803        if metadata.mode == BgMode::Pty {
4804            return None;
4805        }
4806        let mut buffer = BgBuffer::registered(paths, BgMode::Pipes);
4807        let disk_truncation = buffer.enforce_terminal_cap();
4808        Some(self.render_terminal_output(metadata, &buffer, disk_truncation, Some(paths)))
4809    }
4810
4811    fn enqueue_completion_from_parts(
4812        &self,
4813        metadata: &PersistedTask,
4814        buffer: Option<&BgBuffer>,
4815        paths: Option<&TaskPaths>,
4816        emit_frame: bool,
4817        terminal_render: Option<&TerminalOutputCache>,
4818    ) {
4819        // Only the terminal-state guard prevents double-recording here. The
4820        // `completion_delivered` flag is NOT used to gate compression-event
4821        // recording, because `mark_terminal` flips `completion_delivered=true`
4822        // immediately for tasks with `notify_on_completion=false` (foreground
4823        // bash polled via `bash_status`, which is the common case). Pre-emptive
4824        // delivery flagging is correct for the push-frame queue (suppresses
4825        // duplicate user-visible notifications) but would silently skip the
4826        // database insert below. Compression event recording is idempotent at
4827        // the DB layer (unique on harness+session+task_id), so re-entry is
4828        // safe; the dedupe-by-queue check stays for the push frame side.
4829        if !metadata.status.is_terminal() {
4830            return;
4831        }
4832
4833        let owned_buffer = if buffer.is_none() && metadata.mode != BgMode::Pty {
4834            paths.map(|paths| BgBuffer::registered(paths, BgMode::Pipes))
4835        } else {
4836            None
4837        };
4838        let render_buffer = buffer.or(owned_buffer.as_ref());
4839        let owned_render = if terminal_render.is_none() {
4840            render_buffer.map(|buffer| {
4841                let mut capped_buffer = buffer.clone();
4842                let disk_truncation = capped_buffer.enforce_terminal_cap();
4843                self.render_terminal_output(metadata, &capped_buffer, disk_truncation, paths)
4844            })
4845        } else {
4846            None
4847        };
4848        let render = terminal_render.or(owned_render.as_ref());
4849
4850        // Completion reminders use the already-rendered terminal output and a
4851        // smaller, exit-aware head+tail cap. They never invoke the compressor
4852        // themselves.
4853        let (mut output_preview, output_truncated) = render
4854            .map(|cache| completion_preview_for_cache(cache, metadata.exit_code))
4855            .unwrap_or_else(|| (String::new(), false));
4856        if metadata.status == BgTaskStatus::FateUnknown {
4857            if let Some(reason) = metadata.status_reason.as_deref() {
4858                output_preview = if output_preview.is_empty() {
4859                    reason.to_string()
4860                } else {
4861                    format!("{reason}\n{output_preview}")
4862                };
4863            }
4864        }
4865        let bash_output_list_envelope =
4866            render.and_then(|cache| append_bash_output_envelope(cache, &mut output_preview));
4867
4868        let token_counts = self.completion_token_counts(
4869            metadata,
4870            buffer,
4871            paths,
4872            render.map(|render| render.output_preview.as_str()),
4873        );
4874        let completion = BgCompletion {
4875            task_id: metadata.task_id.clone(),
4876            session_id: metadata.session_id.clone(),
4877            status: metadata.status.clone(),
4878            exit_code: metadata.exit_code,
4879            command: metadata.command.clone(),
4880            output_preview,
4881            bash_output_list_envelope,
4882            output_truncated,
4883            original_tokens: token_counts.original_tokens,
4884            compressed_tokens: token_counts.compressed_tokens,
4885            tokens_skipped: token_counts.tokens_skipped,
4886            status_reason: metadata.status_reason.clone(),
4887            live_descendants: metadata.live_descendants.clone(),
4888            live_descendants_omitted: metadata.live_descendants_omitted,
4889            live_descendants_summary: live_descendants_summary(metadata),
4890        };
4891
4892        // Record the compression event BEFORE the push-frame dedupe. Event
4893        // recording has its own idempotency at the DB layer (unique key on
4894        // harness+session+task_id), so it's safe to attempt for every
4895        // terminal-state finalize. Critically, this path runs even when
4896        // `completion_delivered=true` was pre-set by `mark_terminal` for
4897        // foreground bash (`notify_on_completion=false`) — which is the common
4898        // case for OpenCode/Pi `bash` tool calls. Previously this code lived
4899        // after the dedupe guard and never fired for foreground tasks, which
4900        // meant compression accounting was effectively dead for >99% of
4901        // real-world bash usage.
4902        self.record_compression_event_if_applicable(metadata, &token_counts);
4903
4904        // A session ack can arrive through another root actor. Reconcile with
4905        // the shared rows before task-local watch flags decide terminal routing.
4906        self.sync_memory_watches_from_persistence(&metadata.task_id);
4907        let (watch_controlled, watch_matched) = self.task_watch_state(&metadata.task_id);
4908        if watch_controlled {
4909            if !watch_matched && !metadata.completion_delivered {
4910                self.record_bash_watch_exit(&completion, emit_frame);
4911            }
4912            // Clear only in-memory control state; the pending SQLite watch row
4913            // survives until ack, GC, or re-arm settlement.
4914            self.clear_task_watch_state(&metadata.task_id);
4915            return;
4916        }
4917
4918        // Push-frame queue is gated on `completion_delivered` so foreground
4919        // bash with `notify_on_completion=false` does not leak a user-visible
4920        // completion notification. `mark_terminal` pre-sets
4921        // `completion_delivered=true` for those tasks; honoring it here keeps
4922        // the suppression invariant the test
4923        // `no_notify_foreground_poll_completion_does_not_enqueue_completion`
4924        // asserts. The compression-event recording above intentionally runs
4925        // before this gate so foreground bash still contributes to the
4926        // session/project aggregates.
4927        if metadata.completion_delivered {
4928            return;
4929        }
4930
4931        // Push-frame queue dedupe stays per-task to prevent duplicate
4932        // user-visible completion notifications.
4933        let pushed = if let Ok(mut completions) = self.inner.completions.lock() {
4934            if completions
4935                .iter()
4936                .any(|existing| existing.task_id == metadata.task_id)
4937            {
4938                false
4939            } else {
4940                completions.push_back(completion.clone());
4941                true
4942            }
4943        } else {
4944            false
4945        };
4946
4947        if pushed && emit_frame {
4948            self.emit_bash_completed(completion);
4949        }
4950    }
4951
4952    fn record_compression_event_if_applicable(
4953        &self,
4954        metadata: &PersistedTask,
4955        token_counts: &CompletionTokenCounts,
4956    ) {
4957        if metadata.mode == BgMode::Pty {
4958            return;
4959        }
4960
4961        let (original_tokens, compressed_tokens, original_bytes, compressed_bytes) = match (
4962            token_counts.original_tokens,
4963            token_counts.compressed_tokens,
4964            token_counts.original_bytes,
4965            token_counts.compressed_bytes,
4966        ) {
4967            (
4968                Some(original_tokens),
4969                Some(compressed_tokens),
4970                Some(original_bytes),
4971                Some(compressed_bytes),
4972            ) => (
4973                original_tokens,
4974                compressed_tokens,
4975                original_bytes,
4976                compressed_bytes,
4977            ),
4978            _ => {
4979                crate::slog_warn!(
4980                    "compression event skipped for {}: token counts unavailable (likely spill file missing or unreadable)",
4981                    metadata.task_id
4982                );
4983                return;
4984            }
4985        };
4986
4987        let pool = self.inner.db_pool.read().ok().and_then(|slot| slot.clone());
4988        let Some(pool) = pool else {
4989            crate::slog_warn!(
4990                "compression event skipped for {}: db_pool not initialized — was configure run?",
4991                metadata.task_id
4992            );
4993            return;
4994        };
4995        let harness = self
4996            .inner
4997            .db_harness
4998            .read()
4999            .ok()
5000            .and_then(|slot| slot.clone());
5001        let Some(harness) = harness else {
5002            crate::slog_warn!(
5003                "compression event insert skipped for {}: harness not configured",
5004                metadata.task_id
5005            );
5006            return;
5007        };
5008
5009        let project_root = metadata
5010            .project_root
5011            .as_deref()
5012            .unwrap_or(&metadata.workdir);
5013        let project_key = crate::path_identity::project_scope_key(project_root);
5014        let row = crate::db::compression_events::CompressionEventRow {
5015            harness: &harness,
5016            session_id: Some(&metadata.session_id),
5017            project_key: &project_key,
5018            tool: "bash",
5019            task_id: Some(&metadata.task_id),
5020            command: Some(&metadata.command),
5021            compressor: if metadata.compressed {
5022                "registry"
5023            } else {
5024                "none"
5025            },
5026            original_bytes,
5027            compressed_bytes,
5028            original_tokens,
5029            compressed_tokens,
5030            created_at: unix_millis() as i64,
5031        };
5032
5033        let conn = match pool.lock() {
5034            Ok(conn) => conn,
5035            Err(_) => {
5036                crate::slog_warn!(
5037                    "compression event insert failed for {}: db mutex poisoned",
5038                    metadata.task_id
5039                );
5040                return;
5041            }
5042        };
5043        match crate::db::compression_events::insert_compression_event(&conn, &row) {
5044            Ok(Some(row_id)) => {
5045                // The database mutex remains held while the matching warm entries
5046                // advance, so status cannot observe the durable row without its
5047                // in-process aggregate delta.
5048                self.inner
5049                    .compression_aggregates
5050                    .record_successful_insert(&conn, &row, row_id);
5051                // DEBUG-level: each foreground bash call records one of these,
5052                // which clutters info-level logs without adding diagnostic value.
5053                // Aggregate totals are visible via the status RPC / TUI sidebar.
5054                crate::slog_debug!(
5055                    "compression event recorded for {} (project={}, session={}, {} → {} tokens)",
5056                    metadata.task_id,
5057                    project_key,
5058                    metadata.session_id,
5059                    original_tokens,
5060                    compressed_tokens
5061                );
5062            }
5063            Ok(None) => {
5064                crate::slog_debug!(
5065                    "duplicate compression event ignored for {} (project={}, session={})",
5066                    metadata.task_id,
5067                    project_key,
5068                    metadata.session_id
5069                );
5070            }
5071            Err(error) => {
5072                crate::slog_warn!(
5073                    "compression event insert failed for {}: {}",
5074                    metadata.task_id,
5075                    error
5076                );
5077            }
5078        }
5079    }
5080
5081    fn emit_bash_pattern_match(&self, session_id: &str, pattern_match: PatternMatch) {
5082        if !self.originating_session_has_live_route(session_id) {
5083            return;
5084        }
5085        let Ok(progress_sender) = self
5086            .inner
5087            .progress_sender
5088            .lock()
5089            .map(|sender| sender.clone())
5090        else {
5091            return;
5092        };
5093        if let Some(sender) = progress_sender.as_ref() {
5094            let frame = if pattern_match.match_text == WATCH_TASK_EXIT_TEXT {
5095                BashPatternMatchFrame::task_exit(
5096                    pattern_match.task_id,
5097                    session_id.to_string(),
5098                    pattern_match.match_text,
5099                    pattern_match.context,
5100                )
5101            } else {
5102                BashPatternMatchFrame::new(
5103                    pattern_match.task_id,
5104                    session_id.to_string(),
5105                    pattern_match.watch_id,
5106                    pattern_match.match_text,
5107                    pattern_match.match_offset,
5108                    pattern_match.context,
5109                    pattern_match.once,
5110                )
5111            };
5112            sender(PushFrame::BashPatternMatch(frame));
5113        }
5114    }
5115
5116    fn emit_bash_watch_erased(&self, session_id: &str, task_id: &str, watch_id: &str) {
5117        if !self.originating_session_has_live_route(session_id) {
5118            return;
5119        }
5120        let Ok(progress_sender) = self
5121            .inner
5122            .progress_sender
5123            .lock()
5124            .map(|sender| sender.clone())
5125        else {
5126            return;
5127        };
5128        let Some(sender) = progress_sender.as_ref() else {
5129            return;
5130        };
5131        sender(PushFrame::BashPatternMatch(
5132            BashPatternMatchFrame::watch_target_erased(
5133                task_id,
5134                session_id,
5135                watch_id,
5136                WATCH_TARGET_ERASED_TEXT,
5137                WATCH_TARGET_ERASED_CONTEXT,
5138            ),
5139        ));
5140    }
5141
5142    fn record_bash_watch_exit(&self, completion: &BgCompletion, emit_frame: bool) {
5143        let status = completion_status_text(&completion.status, completion.exit_code);
5144        let preview = completion.output_preview.trim_end();
5145        let context = if preview.is_empty() {
5146            format!("task {} exited ({status})", completion.task_id)
5147        } else {
5148            format!(
5149                "task {} exited ({status})
5150{preview}",
5151                completion.task_id
5152            )
5153        };
5154        let frame = BashPatternMatchFrame::task_exit(
5155            completion.task_id.clone(),
5156            completion.session_id.clone(),
5157            format!("exited ({status})"),
5158            context,
5159        );
5160        self.persist_bash_watch_exit(&frame);
5161        if emit_frame {
5162            self.emit_bash_watch_exit(frame);
5163        }
5164    }
5165
5166    fn persist_bash_watch_exit(&self, frame: &BashPatternMatchFrame) {
5167        let Some((harness, pool)) = self.db_harness_and_pool() else {
5168            return;
5169        };
5170        let Ok(conn) = pool.lock() else {
5171            return;
5172        };
5173        let Ok(mut rows) = crate::db::bash_watches::list_bash_pattern_watches_for_task(
5174            &conn,
5175            &harness,
5176            &frame.session_id,
5177            &frame.task_id,
5178        ) else {
5179            return;
5180        };
5181        let Some(mut row) = rows
5182            .iter()
5183            .position(|row| row.match_text.as_deref() == Some(WATCH_TASK_EXIT_TEXT))
5184            .map(|index| rows.swap_remove(index))
5185            .or_else(|| rows.into_iter().next())
5186        else {
5187            return;
5188        };
5189        row.scanning = false;
5190        row.pending_match = true;
5191        row.match_text = Some(WATCH_TASK_EXIT_TEXT.to_string());
5192        row.match_offset = Some(0);
5193        row.match_context = Some(frame.context.clone());
5194        if let Err(error) = crate::db::bash_watches::upsert_bash_pattern_watch(&conn, &row) {
5195            crate::slog_warn!(
5196                "persist bash watch task-exit failed for {}/{}: {error}",
5197                frame.task_id,
5198                row.watch_id
5199            );
5200        }
5201    }
5202
5203    fn emit_bash_watch_exit(&self, frame: BashPatternMatchFrame) {
5204        if !self.originating_session_has_live_route(&frame.session_id) {
5205            return;
5206        }
5207        let Ok(progress_sender) = self
5208            .inner
5209            .progress_sender
5210            .lock()
5211            .map(|sender| sender.clone())
5212        else {
5213            return;
5214        };
5215        let Some(sender) = progress_sender.as_ref() else {
5216            return;
5217        };
5218        sender(PushFrame::BashPatternMatch(frame));
5219    }
5220
5221    fn emit_bash_completed(&self, completion: BgCompletion) {
5222        if !self.originating_session_has_live_route(&completion.session_id) {
5223            return;
5224        }
5225        let Ok(progress_sender) = self
5226            .inner
5227            .progress_sender
5228            .lock()
5229            .map(|sender| sender.clone())
5230        else {
5231            return;
5232        };
5233        let Some(sender) = progress_sender.as_ref() else {
5234            return;
5235        };
5236        // Clone the callback out of the registry mutex before writing to stdout;
5237        // otherwise a blocked push-frame write could pin the mutex and starve
5238        // unrelated progress-sender updates.
5239        // Bg task transitions are discovered by the watchdog thread, so the
5240        // sender is shared behind a Mutex. It still uses the same stdout writer
5241        // closure as foreground progress frames, preserving the existing lock/
5242        // flush behavior in main.rs.
5243        let mut frame = BashCompletedFrame::new(
5244            completion.task_id,
5245            completion.session_id,
5246            completion.status,
5247            completion.exit_code,
5248            completion.command,
5249            completion.output_preview,
5250            completion.output_truncated,
5251            completion.original_tokens,
5252            completion.compressed_tokens,
5253            completion.tokens_skipped,
5254        );
5255        frame.bash_output_list_envelope = completion.bash_output_list_envelope;
5256        frame.status_reason = completion.status_reason;
5257        frame.live_descendants = completion.live_descendants;
5258        frame.live_descendants_omitted = completion.live_descendants_omitted;
5259        frame.live_descendants_summary = completion.live_descendants_summary;
5260        sender(PushFrame::BashCompleted(frame));
5261    }
5262
5263    fn completion_token_counts(
5264        &self,
5265        metadata: &PersistedTask,
5266        buffer: Option<&BgBuffer>,
5267        paths: Option<&TaskPaths>,
5268        rendered_output: Option<&str>,
5269    ) -> CompletionTokenCounts {
5270        if metadata.mode == BgMode::Pty {
5271            return CompletionTokenCounts::skipped();
5272        }
5273
5274        let raw = match buffer {
5275            Some(buffer) => buffer.read_for_token_count(TOKENIZE_CAP_BYTES_PER_STREAM),
5276            None => paths
5277                .map(|paths| {
5278                    read_for_token_count_from_disk(metadata, paths, TOKENIZE_CAP_BYTES_PER_STREAM)
5279                })
5280                .unwrap_or(TokenCountInput::Skipped),
5281        };
5282
5283        let TokenCountInput::Text(raw_output) = raw else {
5284            return CompletionTokenCounts::skipped();
5285        };
5286
5287        let original_tokens = token_count_u32(&raw_output);
5288        let original_bytes = raw_output.len() as i64;
5289        let compressed_output = rendered_output.unwrap_or(&raw_output);
5290        let compressed_tokens = token_count_u32(compressed_output);
5291        let compressed_bytes = compressed_output.len() as i64;
5292        CompletionTokenCounts {
5293            original_tokens: Some(original_tokens),
5294            compressed_tokens: Some(compressed_tokens),
5295            original_bytes: Some(original_bytes),
5296            compressed_bytes: Some(compressed_bytes),
5297            tokens_skipped: false,
5298        }
5299    }
5300
5301    pub(crate) fn maybe_emit_long_running_reminder(&self, task: &Arc<BgTask>) {
5302        if !self
5303            .inner
5304            .long_running_reminder_enabled
5305            .load(Ordering::SeqCst)
5306        {
5307            return;
5308        }
5309        let interval_ms = self
5310            .inner
5311            .long_running_reminder_interval_ms
5312            .load(Ordering::SeqCst);
5313        if interval_ms == 0 {
5314            return;
5315        }
5316        let interval = Duration::from_millis(interval_ms);
5317        let now = Instant::now();
5318        let Ok(mut last_reminder_at) = task.last_reminder_at.lock() else {
5319            return;
5320        };
5321        let since = last_reminder_at.unwrap_or(task.started);
5322        if now.duration_since(since) < interval {
5323            return;
5324        }
5325        let command = task
5326            .state
5327            .lock()
5328            .map(|state| state.metadata.command.clone())
5329            .unwrap_or_default();
5330        *last_reminder_at = Some(now);
5331        self.emit_bash_long_running(BashLongRunningFrame::new(
5332            task.task_id.clone(),
5333            task.session_id.clone(),
5334            command,
5335            task.started.elapsed().as_millis() as u64,
5336        ));
5337    }
5338
5339    fn emit_bash_long_running(&self, frame: BashLongRunningFrame) {
5340        if !self.originating_session_has_live_route(&frame.session_id) {
5341            return;
5342        }
5343        let Ok(progress_sender) = self
5344            .inner
5345            .progress_sender
5346            .lock()
5347            .map(|sender| sender.clone())
5348        else {
5349            return;
5350        };
5351        if let Some(sender) = progress_sender.as_ref() {
5352            sender(PushFrame::BashLongRunning(frame));
5353        }
5354    }
5355
5356    fn task(&self, task_id: &str) -> Option<Arc<BgTask>> {
5357        validate_task_id(task_id).ok()?;
5358        self.inner
5359            .tasks
5360            .lock()
5361            .ok()
5362            .and_then(|tasks| tasks.get(task_id).cloned())
5363    }
5364
5365    fn task_for_session(&self, task_id: &str, session_id: &str) -> Option<Arc<BgTask>> {
5366        self.task(task_id)
5367            .filter(|task| task.session_id == session_id)
5368    }
5369
5370    pub fn try_health_counts(&self) -> Option<BgTaskHealthCounts> {
5371        let running = self
5372            .inner
5373            .tasks
5374            .try_lock()
5375            .ok()
5376            .map(|tasks| tasks.values().filter(|task| task.is_running()).count())?;
5377        let pending_completions = self.inner.completions.try_lock().ok().map(|q| q.len())?;
5378        Some(BgTaskHealthCounts {
5379            running,
5380            pending_completions,
5381        })
5382    }
5383
5384    /// Count background task PIDs that are still alive without keeping registry
5385    /// locks across the OS liveness probes used by the lifecycle health rollup.
5386    pub(crate) fn detached_live_process_count(&self) -> usize {
5387        let Some(pids) = self.inner.tasks.try_lock().ok().map(|tasks| {
5388            tasks
5389                .values()
5390                .filter_map(|task| {
5391                    task.state
5392                        .try_lock()
5393                        .ok()
5394                        .and_then(|state| state.metadata.child_pid)
5395                })
5396                .collect::<Vec<_>>()
5397        }) else {
5398            return 0;
5399        };
5400        pids.into_iter()
5401            .filter(|pid| is_process_alive(*pid))
5402            .count()
5403    }
5404
5405    /// Estimate resident bash output caches without reading disk-backed task
5406    /// streams. Spill files are deliberately excluded because they do not
5407    /// occupy the daemon heap.
5408    pub fn estimated_memory(&self) -> crate::memory::MemoryEstimate {
5409        let tasks = match self.inner.tasks.try_lock() {
5410            Ok(tasks) => tasks.values().cloned().collect::<Vec<_>>(),
5411            Err(_) => return crate::memory::MemoryEstimate::busy(),
5412        };
5413        let mut bytes = 0u64;
5414        let mut terminal_output_caches = 0usize;
5415        let mut sessions = HashSet::new();
5416        for task in &tasks {
5417            sessions.insert(task.session_id.clone());
5418            let state = match task.state.try_lock() {
5419                Ok(state) => state,
5420                Err(_) => return crate::memory::MemoryEstimate::busy(),
5421            };
5422            if let Some(cache) = state.terminal_output_cache.as_ref() {
5423                terminal_output_caches = terminal_output_caches.saturating_add(1);
5424                bytes = bytes.saturating_add(terminal_output_cache_estimated_bytes(cache));
5425            }
5426        }
5427        let completion_count = match self.inner.completions.try_lock() {
5428            Ok(completions) => {
5429                for completion in completions.iter() {
5430                    sessions.insert(completion.session_id.clone());
5431                    bytes = bytes.saturating_add(completion_estimated_bytes(completion));
5432                }
5433                completions.len()
5434            }
5435            Err(_) => return crate::memory::MemoryEstimate::busy(),
5436        };
5437
5438        crate::memory::MemoryEstimate::estimated(bytes)
5439            .count("tasks", tasks.len())
5440            .count("sessions", sessions.len())
5441            .count("terminal_output_caches", terminal_output_caches)
5442            .count("completion_caches", completion_count)
5443            .count_u64("output_ring_bytes", 0)
5444    }
5445
5446    fn running_count(&self) -> usize {
5447        self.inner
5448            .tasks
5449            .lock()
5450            .map(|tasks| tasks.values().filter(|task| task.is_running()).count())
5451            .unwrap_or(0)
5452    }
5453
5454    fn start_watchdog(&self) {
5455        if !self.inner.watchdog_started.swap(true, Ordering::SeqCst) {
5456            super::watchdog::start(self.clone());
5457        }
5458    }
5459
5460    #[cfg(test)]
5461    pub fn task_json_path(&self, task_id: &str, session_id: &str) -> Option<PathBuf> {
5462        self.task_for_session(task_id, session_id)
5463            .map(|task| task.paths.json.clone())
5464    }
5465
5466    #[cfg(test)]
5467    pub fn task_exit_path(&self, task_id: &str, session_id: &str) -> Option<PathBuf> {
5468        self.task_for_session(task_id, session_id)
5469            .map(|task| task.paths.exit.clone())
5470    }
5471}
5472
5473#[cfg(unix)]
5474fn should_capture_pipeline_status(
5475    spawn_plan: &SpawnPlan,
5476    has_pipeline: bool,
5477    shell: &Path,
5478) -> bool {
5479    if spawn_plan.is_native_launcher() {
5480        // Landlock closes fd 5 and above before exec; sandboxed tasks therefore
5481        // cannot safely pass CHILD_PIPE_STATUS_FD to the payload wrapper.
5482        return false;
5483    }
5484    has_pipeline && super::process::pipeline_shell_kind(shell).is_some()
5485}
5486
5487fn canonical_artifact_root(paths: &TaskPaths) -> PathBuf {
5488    fs::canonicalize(&paths.io_dir).unwrap_or_else(|_| paths.io_dir.clone())
5489}
5490
5491fn restart_fate_unknown_reason(metadata: &PersistedTask, paths: &TaskPaths) -> String {
5492    let output = match metadata.mode {
5493        BgMode::Pipes => &paths.stdout,
5494        BgMode::Pty => &paths.pty,
5495    };
5496    format!(
5497        "task {}: daemon restarted, process fate unknown, last output at {}",
5498        metadata.task_id,
5499        output.display()
5500    )
5501}
5502
5503/// Append the pipeline-failure note after compression and output capping. The
5504/// note is intentionally derived from the same scanner used before spawning;
5505/// the status file only supplies numeric results for those already-known
5506/// segments.
5507fn append_pipeline_warning(
5508    cache: &mut TerminalOutputCache,
5509    metadata: &PersistedTask,
5510    paths: Option<&TaskPaths>,
5511) {
5512    if metadata.exit_code != Some(0) {
5513        return;
5514    }
5515    let Some(paths) = paths else {
5516        return;
5517    };
5518    if metadata.pipeline_segments.len() < 2 {
5519        return;
5520    }
5521    let mut status_file = match open_task_artifact(paths, TaskArtifact::PipelineStatus) {
5522        Ok(file) => file,
5523        Err(_) => {
5524            let Some(shell) = metadata.pipeline_status_unavailable.as_deref() else {
5525                return;
5526            };
5527            let footer = format!(
5528                "note: pipeline status unavailable under {shell}; an upstream failure may be masked by the final segment's exit code."
5529            );
5530            if cache.output_preview.trim().is_empty() {
5531                cache.output_preview = footer;
5532            } else {
5533                cache.output_preview = format!("{}\n{footer}", cache.output_preview.trim_end());
5534            }
5535            return;
5536        }
5537    };
5538    let Ok(status_bytes) = status_file.read_all() else {
5539        return;
5540    };
5541    let Some(statuses) = String::from_utf8_lossy(&status_bytes)
5542        .lines()
5543        .map(|line| line.trim().parse::<i32>().ok())
5544        .collect::<Option<Vec<_>>>()
5545    else {
5546        return;
5547    };
5548    if statuses.len() != metadata.pipeline_segments.len() {
5549        return;
5550    }
5551    let Some((failing_index, failing_code)) = statuses
5552        .iter()
5553        .enumerate()
5554        .take(statuses.len().saturating_sub(1))
5555        .find(|(_, code)| **code != 0)
5556        .map(|(index, code)| (index, *code))
5557    else {
5558        return;
5559    };
5560    let Some(final_segment) = metadata.pipeline_segments.last() else {
5561        return;
5562    };
5563    let failing_segment = &metadata.pipeline_segments[failing_index];
5564    let footer = format!(
5565        "note: `{}` (segment {} of {}) exited {}; the pipeline's exit code is `{}`'s.",
5566        failing_segment,
5567        failing_index + 1,
5568        metadata.pipeline_segments.len(),
5569        failing_code,
5570        final_segment,
5571    );
5572    if cache.output_preview.trim().is_empty() {
5573        cache.output_preview = footer;
5574    } else {
5575        cache.output_preview = format!("{}\n{}", cache.output_preview.trim_end(), footer,);
5576    }
5577}
5578
5579/// Normalize pipes for text rendering without changing their byte-exact artifacts.
5580/// PTY output bypasses this path because its vt100 renderer owns terminal state.
5581fn normalize_piped_display_output(text: &mut String) {
5582    if !text.contains('\r') {
5583        return;
5584    }
5585
5586    let mut rendered = String::with_capacity(text.len());
5587    let mut line = Vec::new();
5588    let mut column = 0;
5589    let mut chars = text.chars().peekable();
5590
5591    while let Some(character) = chars.next() {
5592        match character {
5593            '\r' if chars.peek() == Some(&'\n') => {
5594                chars.next();
5595                for character in &line {
5596                    rendered.push(*character);
5597                }
5598                rendered.push('\n');
5599                line.clear();
5600                column = 0;
5601            }
5602            '\r' => column = 0,
5603            '\n' => {
5604                for character in &line {
5605                    rendered.push(*character);
5606                }
5607                rendered.push('\n');
5608                line.clear();
5609                column = 0;
5610            }
5611            character => {
5612                if column < line.len() {
5613                    line[column] = character;
5614                } else {
5615                    line.resize(column, ' ');
5616                    line.push(character);
5617                }
5618                column += 1;
5619            }
5620        }
5621    }
5622
5623    for character in &line {
5624        rendered.push(*character);
5625    }
5626    *text = rendered;
5627}
5628
5629fn render_compressed_with_recovery(
5630    buffer: &BgBuffer,
5631    mut compressed: CompressionResult,
5632    input_truncated: bool,
5633    disk_truncation: DiskTruncation,
5634    artifact_access: ArtifactRecoveryAccess,
5635) -> TerminalOutputCache {
5636    // Preserve a single canonical trailing newline. A bare `.trim_end()` strips
5637    // the legitimate final newline that `echo` and most commands emit, so
5638    // agent-facing output diverged from native bash ("hello" vs "hello\n") and
5639    // broke the no-JSON-envelope contract. Collapse excess trailing blank lines
5640    // to one, but keep that one when the content had a trailing newline. NOTE:
5641    // the check must read the ORIGINAL text — strip_plain_truncation_marker_lines
5642    // rebuilds via `.lines().join("\n")`, which itself drops the trailing newline.
5643    let compression_input_line_count = compressed.input_line_count;
5644    let had_trailing_newline = compressed.text.ends_with('\n');
5645    let mut text = strip_plain_truncation_marker_lines(&compressed.text)
5646        .trim_end()
5647        .to_string();
5648    if had_trailing_newline && !text.is_empty() {
5649        text.push('\n');
5650    }
5651    compressed.text = text;
5652
5653    let output_path = buffer.output_path().map(|path| path.display().to_string());
5654    let stderr_path = buffer.stderr_path().map(|path| path.display().to_string());
5655    let include_stderr_path = buffer.stream_len(StreamKind::Stderr) > 0;
5656    let mut recovery = RecoveryContext {
5657        dropped_by_class: compressed.dropped_by_class,
5658        had_inner_drop: compressed.had_inner_drop,
5659        offset_hint_eligible: compressed.offset_hint_eligible,
5660        offset_start_line: compressed.offset_start_line,
5661        byte_truncated: input_truncated,
5662        disk_truncated_prefix_bytes: disk_truncation.total_prefix_bytes(),
5663        output_path: output_path.clone(),
5664        stderr_path: stderr_path.clone(),
5665        include_stderr_path,
5666        artifact_access: artifact_access.clone(),
5667    };
5668
5669    let (output_preview, output_truncated) =
5670        render_body_with_recovery_marker(&compressed.text, &mut recovery);
5671    TerminalOutputCache {
5672        output_preview,
5673        output_truncated,
5674        compression_input_line_count: Some(compression_input_line_count),
5675        kind: TerminalOutputKind::Compressed,
5676        output_path,
5677        stderr_path,
5678        artifact_access,
5679        recovery: Some(recovery),
5680    }
5681}
5682
5683fn render_body_with_recovery_marker(body: &str, recovery: &mut RecoveryContext) -> (String, bool) {
5684    render_body_with_recovery_marker_at_cap(
5685        body,
5686        recovery,
5687        FINAL_OUTPUT_CAP_BYTES,
5688        cap_final_output,
5689        cap_final_output_with_marker,
5690    )
5691}
5692
5693fn render_raw_body_with_recovery_marker(
5694    body: &str,
5695    recovery: &mut RecoveryContext,
5696) -> (String, bool) {
5697    render_body_with_recovery_marker_at_cap(
5698        body,
5699        recovery,
5700        RAW_PASSTHROUGH_CAP_BYTES,
5701        |input| {
5702            super::output::cap_head_tail(
5703                input,
5704                RAW_PASSTHROUGH_CAP_BYTES,
5705                RAW_PASSTHROUGH_HEAD_BYTES,
5706                RAW_PASSTHROUGH_TAIL_BYTES,
5707            )
5708        },
5709        |input, marker| {
5710            super::output::cap_head_tail_with_marker(
5711                input,
5712                RAW_PASSTHROUGH_CAP_BYTES,
5713                RAW_PASSTHROUGH_HEAD_BYTES,
5714                RAW_PASSTHROUGH_TAIL_BYTES,
5715                marker,
5716            )
5717        },
5718    )
5719}
5720
5721fn render_body_with_recovery_marker_at_cap<F, G>(
5722    body: &str,
5723    recovery: &mut RecoveryContext,
5724    cap_bytes: usize,
5725    cap_plain: F,
5726    cap_with_marker: G,
5727) -> (String, bool)
5728where
5729    F: Fn(&str) -> super::output::CappedText,
5730    G: Fn(&str, &str) -> super::output::CappedText,
5731{
5732    let needs_marker = recovery.has_visible_drop();
5733    if body.len() > cap_bytes {
5734        recovery.byte_truncated = true;
5735        if let Some(marker) = recovery_marker(recovery) {
5736            let capped = cap_with_marker(body, &marker);
5737            return (capped.text, true);
5738        }
5739        let capped = cap_plain(body);
5740        return (capped.text, capped.truncated || needs_marker);
5741    }
5742
5743    if !needs_marker {
5744        return (body.to_string(), false);
5745    }
5746
5747    let Some(marker) = recovery_marker(recovery) else {
5748        return (body.to_string(), true);
5749    };
5750    let with_marker = append_recovery_marker(body, &marker);
5751    if with_marker.len() <= cap_bytes {
5752        return (with_marker, true);
5753    }
5754
5755    recovery.byte_truncated = true;
5756    let marker = recovery_marker(recovery).unwrap_or(marker);
5757    let capped = cap_with_marker(body, &marker);
5758    (capped.text, true)
5759}
5760
5761fn append_recovery_marker(body: &str, marker: &str) -> String {
5762    if body.is_empty() {
5763        return marker.to_string();
5764    }
5765    let mut output = body.trim_end().to_string();
5766    output.push('\n');
5767    output.push_str(marker);
5768    output
5769}
5770
5771fn recovery_marker(recovery: &RecoveryContext) -> Option<String> {
5772    let mut parts = Vec::new();
5773    for (class, count) in &recovery.dropped_by_class {
5774        let label = if *count == 1 {
5775            class.singular()
5776        } else {
5777            class.plural()
5778        };
5779        parts.push(format!("+{count} more {label}"));
5780    }
5781    if recovery.byte_truncated {
5782        parts.push("truncated output".to_string());
5783    }
5784    let disk_truncated_prefix_bytes = recovery.disk_truncated_prefix_bytes;
5785    if disk_truncated_prefix_bytes > 0 {
5786        parts.push(format!(
5787            "truncated {disk_truncated_prefix_bytes} bytes from saved output prefix"
5788        ));
5789    } else if recovery.had_inner_drop && parts.is_empty() {
5790        parts.push("omitted output".to_string());
5791    }
5792
5793    if parts.is_empty() {
5794        return None;
5795    }
5796
5797    let hint = recovery_hint(recovery);
5798    Some(format!("[{}; {hint}]", parts.join(", ")))
5799}
5800
5801fn bash_status_recovery_hint(access: &ArtifactRecoveryAccess) -> String {
5802    let task_id = serde_json::to_string(&access.task_id)
5803        .unwrap_or_else(|_| format!("\"{}\"", access.task_id));
5804    format!("use bash_status({{taskId: {task_id}}})")
5805}
5806
5807fn recovery_hint(recovery: &RecoveryContext) -> String {
5808    if !recovery.artifact_access.readable {
5809        return bash_status_recovery_hint(&recovery.artifact_access);
5810    }
5811
5812    // AFT stores stdout/stderr separately and combines them in memory. Class caps,
5813    // middle truncation, and mixed stdout/stderr renders are not line-offset
5814    // portable. Only a single-file contiguous-prefix drop may use `tail -n +N`.
5815    if recovery.offset_hint_eligible
5816        && !recovery.byte_truncated
5817        && recovery.dropped_by_class.is_empty()
5818        && !recovery.include_stderr_path
5819    {
5820        if let (Some(path), Some(line)) =
5821            (recovery.output_path.as_deref(), recovery.offset_start_line)
5822        {
5823            return format!("see remaining: tail -n +{line} {}", quote_path(path));
5824        }
5825    }
5826
5827    let mut paths = Vec::new();
5828    if let Some(path) = recovery.output_path.as_deref() {
5829        paths.push(path);
5830    }
5831    if recovery.include_stderr_path {
5832        if let Some(path) = recovery.stderr_path.as_deref() {
5833            if !paths.contains(&path) {
5834                paths.push(path);
5835            }
5836        }
5837    }
5838
5839    if paths.is_empty() {
5840        return "full output unavailable".to_string();
5841    }
5842
5843    let reads = paths
5844        .into_iter()
5845        .map(|path| format!("read {}", quote_path(path)))
5846        .collect::<Vec<_>>()
5847        .join(" and ");
5848    if recovery.disk_truncated_prefix_bytes > 0 {
5849        format!("retained output: {reads}")
5850    } else {
5851        format!("full output: {reads}")
5852    }
5853}
5854
5855fn strip_plain_truncation_marker_lines(input: &str) -> String {
5856    input
5857        .lines()
5858        .filter(|line| !is_plain_truncation_marker(line.trim()))
5859        .collect::<Vec<_>>()
5860        .join("\n")
5861}
5862
5863fn strip_recovery_marker_lines(input: &str) -> String {
5864    input
5865        .lines()
5866        .filter(|line| !is_recovery_marker(line.trim()))
5867        .collect::<Vec<_>>()
5868        .join("\n")
5869}
5870
5871fn is_plain_truncation_marker(line: &str) -> bool {
5872    let Some(rest) = line.strip_prefix("...<truncated ") else {
5873        return false;
5874    };
5875    let Some(bytes) = rest.strip_suffix(" bytes>...") else {
5876        return false;
5877    };
5878    !bytes.is_empty() && bytes.chars().all(|ch| ch.is_ascii_digit())
5879}
5880
5881fn is_recovery_marker(line: &str) -> bool {
5882    line.starts_with('[')
5883        && line.ends_with(']')
5884        && (line.contains("full output: read ")
5885            || line.contains("retained output: read ")
5886            || line.contains("see remaining: tail -n +")
5887            || line.contains("use bash_status({taskId:")
5888            || line.contains("full output unavailable"))
5889}
5890
5891fn structured_output_pointer(
5892    total_bytes: u64,
5893    output_path: &str,
5894    truncated_prefix_bytes: u64,
5895    artifact_access: &ArtifactRecoveryAccess,
5896) -> String {
5897    if artifact_access.readable {
5898        return if truncated_prefix_bytes > 0 {
5899            retained_json_output_pointer(total_bytes, output_path, truncated_prefix_bytes)
5900        } else {
5901            json_output_pointer(total_bytes, output_path)
5902        };
5903    }
5904
5905    let kb = total_bytes.div_ceil(1024);
5906    let hint = bash_status_recovery_hint(artifact_access);
5907    if truncated_prefix_bytes > 0 {
5908        format!(
5909            "[JSON output {kb} KB; truncated {truncated_prefix_bytes} bytes from saved output prefix; retained output: {hint}]"
5910        )
5911    } else {
5912        format!("[JSON output {kb} KB; full output: {hint}]")
5913    }
5914}
5915
5916fn render_structured_output(
5917    command: &str,
5918    buffer: &BgBuffer,
5919    disk_truncation: DiskTruncation,
5920    artifact_access: ArtifactRecoveryAccess,
5921) -> Option<TerminalOutputCache> {
5922    if !is_gh_structured_command(command) {
5923        return None;
5924    }
5925
5926    let output_path = buffer
5927        .output_path()
5928        .map(|path| path.display().to_string())?;
5929    let stdout_bytes = buffer.stream_len(StreamKind::Stdout);
5930    if stdout_bytes == 0 {
5931        return None;
5932    }
5933
5934    if stdout_bytes > STRUCTURED_OUTPUT_CAP_BYTES as u64 {
5935        if !stream_starts_like_json(buffer, StreamKind::Stdout) {
5936            return None;
5937        }
5938        let output_preview = structured_output_pointer(
5939            stdout_bytes,
5940            &output_path,
5941            disk_truncation.total_prefix_bytes(),
5942            &artifact_access,
5943        );
5944        return Some(TerminalOutputCache {
5945            output_preview,
5946            output_truncated: true,
5947            compression_input_line_count: None,
5948            kind: TerminalOutputKind::Structured,
5949            output_path: Some(output_path),
5950            stderr_path: buffer.stderr_path().map(|path| path.display().to_string()),
5951            artifact_access,
5952            recovery: None,
5953        });
5954    }
5955
5956    let stdout = buffer.read_stream_bounded(StreamKind::Stdout, STRUCTURED_OUTPUT_CAP_BYTES);
5957    if stdout.truncated || !is_structured_body(&stdout.text) {
5958        return None;
5959    }
5960
5961    Some(TerminalOutputCache {
5962        output_preview: stdout.text,
5963        output_truncated: false,
5964        compression_input_line_count: None,
5965        kind: TerminalOutputKind::Structured,
5966        output_path: Some(output_path),
5967        stderr_path: buffer.stderr_path().map(|path| path.display().to_string()),
5968        artifact_access,
5969        recovery: None,
5970    })
5971}
5972
5973fn render_raw_passthrough(
5974    buffer: &BgBuffer,
5975    disk_truncation: DiskTruncation,
5976    artifact_access: ArtifactRecoveryAccess,
5977) -> TerminalOutputCache {
5978    let raw = buffer.read_combined_head_tail(
5979        RAW_PASSTHROUGH_CAP_BYTES,
5980        RAW_PASSTHROUGH_HEAD_BYTES,
5981        RAW_PASSTHROUGH_TAIL_BYTES,
5982    );
5983    let output_path = buffer.output_path().map(|path| path.display().to_string());
5984    let stderr_path = buffer.stderr_path().map(|path| path.display().to_string());
5985    if !raw.truncated && disk_truncation.total_prefix_bytes() == 0 {
5986        return TerminalOutputCache {
5987            output_preview: raw.text,
5988            output_truncated: false,
5989            compression_input_line_count: None,
5990            kind: TerminalOutputKind::Raw,
5991            output_path,
5992            stderr_path,
5993            artifact_access,
5994            recovery: None,
5995        };
5996    }
5997
5998    let include_stderr_path = buffer.stream_len(StreamKind::Stderr) > 0;
5999    let mut recovery = RecoveryContext {
6000        dropped_by_class: BTreeMap::new(),
6001        had_inner_drop: false,
6002        offset_hint_eligible: false,
6003        offset_start_line: None,
6004        byte_truncated: raw.truncated,
6005        disk_truncated_prefix_bytes: disk_truncation.total_prefix_bytes(),
6006        output_path: output_path.clone(),
6007        stderr_path: stderr_path.clone(),
6008        include_stderr_path,
6009        artifact_access: artifact_access.clone(),
6010    };
6011    let (output_preview, output_truncated) =
6012        render_raw_body_with_recovery_marker(&raw.text, &mut recovery);
6013    TerminalOutputCache {
6014        output_preview,
6015        output_truncated,
6016        compression_input_line_count: None,
6017        kind: TerminalOutputKind::Raw,
6018        output_path,
6019        stderr_path,
6020        artifact_access,
6021        recovery: Some(recovery),
6022    }
6023}
6024
6025fn append_bash_output_envelope(
6026    cache: &TerminalOutputCache,
6027    output: &mut String,
6028) -> Option<ListEnvelope> {
6029    cache
6030        .compression_input_line_count
6031        .and_then(|total| crate::list_surfaces::bash::append_envelope_trailer(output, total))
6032}
6033
6034fn completion_preview_for_cache(
6035    cache: &TerminalOutputCache,
6036    exit_code: Option<i32>,
6037) -> (String, bool) {
6038    // Reminder previews are sized by exit status: success gets a short tail,
6039    // failure keeps head+tail context (see output.rs completion caps).
6040    let exit_ok = exit_code == Some(0);
6041    let threshold = completion_preview_threshold(exit_ok);
6042    if cache.kind == TerminalOutputKind::Structured && cache.output_preview.len() > threshold {
6043        if let Some(path) = cache.output_path.as_deref() {
6044            return (
6045                structured_output_pointer(
6046                    cache.output_preview.len() as u64,
6047                    path,
6048                    0,
6049                    &cache.artifact_access,
6050                ),
6051                true,
6052            );
6053        }
6054        return (cache.output_preview.clone(), cache.output_truncated);
6055    }
6056
6057    if let Some(recovery) = cache.recovery.as_ref() {
6058        if cache.output_preview.len() <= threshold {
6059            return (cache.output_preview.clone(), cache.output_truncated);
6060        }
6061        let body = strip_recovery_marker_lines(&cache.output_preview);
6062        let mut completion_recovery = recovery.clone();
6063        completion_recovery.byte_truncated = true;
6064        if let Some(marker) = recovery_marker(&completion_recovery) {
6065            let capped = cap_completion_output_with_marker(&body, &marker, exit_ok);
6066            return (capped.text, true);
6067        }
6068    }
6069
6070    let capped = cap_completion_output(&cache.output_preview, exit_ok);
6071    (capped.text, cache.output_truncated || capped.truncated)
6072}
6073
6074fn is_gh_structured_command(command: &str) -> bool {
6075    let Some(normalized) = crate::compress::plain_command_for_structured_output(command) else {
6076        return false;
6077    };
6078    let tokens = shell_words_for_flags(&normalized);
6079    let Some(head) = tokens.first() else {
6080        return false;
6081    };
6082    let head_name = Path::new(head)
6083        .file_name()
6084        .and_then(|name| name.to_str())
6085        .unwrap_or(head);
6086    if !(head_name == "gh" || head_name.eq_ignore_ascii_case("gh.exe")) {
6087        return false;
6088    }
6089    tokens.iter().any(|token| {
6090        matches!(token.as_str(), "--json" | "--jq" | "--template")
6091            || token.starts_with("--json=")
6092            || token.starts_with("--jq=")
6093            || token.starts_with("--template=")
6094    })
6095}
6096
6097fn shell_words_for_flags(command: &str) -> Vec<String> {
6098    let mut words = Vec::new();
6099    let mut current = String::new();
6100    let mut in_single = false;
6101    let mut in_double = false;
6102    let mut escaped = false;
6103
6104    for ch in command.chars() {
6105        if escaped {
6106            current.push(ch);
6107            escaped = false;
6108            continue;
6109        }
6110        if ch == '\\' && !in_single {
6111            escaped = true;
6112            continue;
6113        }
6114        if ch == '\'' && !in_double {
6115            in_single = !in_single;
6116            continue;
6117        }
6118        if ch == '"' && !in_single {
6119            in_double = !in_double;
6120            continue;
6121        }
6122        if ch.is_whitespace() && !in_single && !in_double {
6123            if !current.is_empty() {
6124                words.push(std::mem::take(&mut current));
6125            }
6126            continue;
6127        }
6128        if matches!(ch, ';' | '&' | '|') && !in_single && !in_double {
6129            if !current.is_empty() {
6130                words.push(std::mem::take(&mut current));
6131            }
6132            continue;
6133        }
6134        current.push(ch);
6135    }
6136    if !current.is_empty() {
6137        words.push(current);
6138    }
6139    words
6140}
6141
6142fn is_structured_body(body: &str) -> bool {
6143    let trimmed = body.trim();
6144    if trimmed.is_empty() {
6145        return false;
6146    }
6147    if serde_json::from_str::<serde_json::Value>(trimmed).is_ok() {
6148        return true;
6149    }
6150
6151    let mut saw_line = false;
6152    for line in trimmed
6153        .lines()
6154        .map(str::trim)
6155        .filter(|line| !line.is_empty())
6156    {
6157        saw_line = true;
6158        if serde_json::from_str::<serde_json::Value>(line).is_err() {
6159            return false;
6160        }
6161    }
6162    saw_line
6163}
6164
6165fn stream_starts_like_json(buffer: &BgBuffer, stream: StreamKind) -> bool {
6166    buffer
6167        .read_stream_bounded(stream, 512)
6168        .text
6169        .chars()
6170        .find(|ch| !ch.is_whitespace())
6171        .is_some_and(|ch| matches!(ch, '{' | '[' | '"' | '-' | '0'..='9' | 't' | 'f' | 'n'))
6172}
6173
6174struct CompletionTokenCounts {
6175    original_tokens: Option<u32>,
6176    compressed_tokens: Option<u32>,
6177    original_bytes: Option<i64>,
6178    compressed_bytes: Option<i64>,
6179    tokens_skipped: bool,
6180}
6181
6182impl CompletionTokenCounts {
6183    fn skipped() -> Self {
6184        Self {
6185            original_tokens: None,
6186            compressed_tokens: None,
6187            original_bytes: None,
6188            compressed_bytes: None,
6189            tokens_skipped: true,
6190        }
6191    }
6192}
6193
6194fn live_descendants_summary(metadata: &PersistedTask) -> Option<String> {
6195    let Some(members) = metadata.live_descendants.as_deref() else {
6196        #[cfg(windows)]
6197        return (metadata.mode == BgMode::Pipes).then(|| {
6198            "live descendant check n/a on Windows (background tasks do not use a Job Object)"
6199                .to_string()
6200        });
6201        #[cfg(not(windows))]
6202        return None;
6203    };
6204    let total = members.len() + metadata.live_descendants_omitted;
6205    if total == 0 {
6206        return None;
6207    }
6208
6209    let mut counts = BTreeMap::<&str, usize>::new();
6210    for member in members {
6211        *counts.entry(member.comm.as_str()).or_default() += 1;
6212    }
6213    let mut processes = counts
6214        .into_iter()
6215        .map(|(comm, count)| {
6216            if count == 1 {
6217                comm.to_string()
6218            } else {
6219                format!("{comm} ×{count}")
6220            }
6221        })
6222        .collect::<Vec<_>>()
6223        .join(", ");
6224    if metadata.live_descendants_omitted > 0 {
6225        processes.push_str(&format!(" +{} more", metadata.live_descendants_omitted));
6226    }
6227    let workload = ["vitest", "jest", "cargo test"]
6228        .into_iter()
6229        .find(|name| metadata.command.contains(name))
6230        .map(|name| format!(" ({name})"))
6231        .unwrap_or_default();
6232    Some(format!(
6233        "{total} live descendants still running: {processes}{workload} — they keep the task's process group; bash_kill({}) stops them",
6234        metadata.task_id
6235    ))
6236}
6237
6238fn completion_status_text(status: &BgTaskStatus, exit_code: Option<i32>) -> String {
6239    match status {
6240        BgTaskStatus::TimedOut => "timed out".to_string(),
6241        BgTaskStatus::Killed => "killed".to_string(),
6242        _ => exit_code
6243            .map(|code| format!("exit {code}"))
6244            .unwrap_or_else(|| format!("{status:?}").to_lowercase()),
6245    }
6246}
6247
6248fn token_count_u32(text: &str) -> u32 {
6249    aft_tokenizer::count_tokens(text)
6250        .try_into()
6251        .unwrap_or(u32::MAX)
6252}
6253
6254impl Default for BgTaskRegistry {
6255    fn default() -> Self {
6256        Self::new(Arc::new(Mutex::new(None)))
6257    }
6258}
6259
6260/// Key for a session's recovery state: the name of its task directory, which
6261/// is what the persisted GC has in hand while walking the storage root.
6262fn session_recovery_key(storage_dir: &Path, session_id: &str) -> Option<String> {
6263    session_tasks_dir(storage_dir, session_id)
6264        .file_name()
6265        .and_then(|name| name.to_str())
6266        .map(str::to_string)
6267}
6268
6269/// True when a validated open lost its race against an atomic replacement
6270/// of the same artifact for every attempt it was allowed. The persistence
6271/// layer reports that case distinctly so the sweep can tell a busy writer
6272/// from a damaged layout.
6273fn concurrently_replaced(error: &std::io::Error) -> bool {
6274    error.kind() == std::io::ErrorKind::Interrupted
6275        && error
6276            .to_string()
6277            .contains(super::persistence::ARTIFACT_CONCURRENTLY_REPLACED)
6278}
6279
6280fn modified_within(path: &Path, grace: Duration) -> bool {
6281    fs::metadata(path)
6282        .and_then(|metadata| metadata.modified())
6283        .ok()
6284        .and_then(|modified| SystemTime::now().duration_since(modified).ok())
6285        .map(|age| age < grace)
6286        .unwrap_or(false)
6287}
6288
6289fn canonicalized_path(path: &Path) -> PathBuf {
6290    fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf())
6291}
6292
6293fn started_instant_from_unix_millis(started_at: u64) -> Instant {
6294    let now_ms = SystemTime::now()
6295        .duration_since(UNIX_EPOCH)
6296        .ok()
6297        .map(|duration| duration.as_millis() as u64)
6298        .unwrap_or(started_at);
6299    let elapsed_ms = now_ms.saturating_sub(started_at);
6300    Instant::now()
6301        .checked_sub(Duration::from_millis(elapsed_ms))
6302        .unwrap_or_else(Instant::now)
6303}
6304
6305fn gc_quarantine(storage_dir: &Path) {
6306    let quarantine_root = storage_dir.join("bash-tasks-quarantine");
6307    let Ok(session_dirs) = fs::read_dir(&quarantine_root) else {
6308        return;
6309    };
6310    for session_entry in session_dirs.flatten() {
6311        let session_quarantine_dir = session_entry.path();
6312        if !session_quarantine_dir.is_dir() {
6313            continue;
6314        }
6315        let entries = match fs::read_dir(&session_quarantine_dir) {
6316            Ok(entries) => entries,
6317            Err(error) => {
6318                crate::slog_warn!(
6319                    "failed to read background task quarantine dir {}: {error}",
6320                    session_quarantine_dir.display()
6321                );
6322                continue;
6323            }
6324        };
6325        for entry in entries.flatten() {
6326            let path = entry.path();
6327            if modified_within(&path, QUARANTINE_GC_GRACE) {
6328                continue;
6329            }
6330            let result = if path.is_dir() {
6331                fs::remove_dir_all(&path)
6332            } else {
6333                fs::remove_file(&path)
6334            };
6335            match result {
6336                Ok(()) => log::debug!(
6337                    "deleted old background task quarantine entry {}",
6338                    path.display()
6339                ),
6340                Err(error) => crate::slog_warn!(
6341                    "failed to delete old background task quarantine entry {}: {error}",
6342                    path.display()
6343                ),
6344            }
6345        }
6346        let _ = fs::remove_dir(&session_quarantine_dir);
6347    }
6348    let _ = fs::remove_dir(&quarantine_root);
6349}
6350
6351fn read_for_token_count_from_disk(
6352    metadata: &PersistedTask,
6353    paths: &TaskPaths,
6354    max_bytes_per_stream: usize,
6355) -> TokenCountInput {
6356    if metadata.mode == BgMode::Pty {
6357        return TokenCountInput::Skipped;
6358    }
6359    // Read up to `max_bytes_per_stream` bytes per stream rather than
6360    // refusing to tokenize anything when the file exceeds the cap.
6361    // Mirror the in-memory `BgBuffer::read_for_token_count` policy
6362    // (see comment there) — large outputs are exactly the tasks that
6363    // benefit most from compression accounting, so silent-skipping
6364    // them defeats the purpose of token tracking.
6365    let stdout = read_file_tail_capped(paths, TaskArtifact::Stdout, max_bytes_per_stream);
6366    let stderr = read_file_tail_capped(paths, TaskArtifact::Stderr, max_bytes_per_stream);
6367    match (stdout, stderr) {
6368        (Ok(stdout), Ok(stderr)) => TokenCountInput::Text(combine_streams(
6369            String::from_utf8_lossy(&stdout).as_ref(),
6370            String::from_utf8_lossy(&stderr).as_ref(),
6371        )),
6372        (Ok(stdout), Err(_)) => TokenCountInput::Text(combine_streams(
6373            String::from_utf8_lossy(&stdout).as_ref(),
6374            "",
6375        )),
6376        (Err(_), Ok(stderr)) => TokenCountInput::Text(combine_streams(
6377            "",
6378            String::from_utf8_lossy(&stderr).as_ref(),
6379        )),
6380        (Err(_), Err(_)) => TokenCountInput::Skipped,
6381    }
6382}
6383
6384fn read_file_tail_capped(
6385    paths: &TaskPaths,
6386    artifact: TaskArtifact,
6387    max_bytes: usize,
6388) -> std::io::Result<Vec<u8>> {
6389    let mut file = open_task_artifact(paths, artifact)?;
6390    file.tail(max_bytes).map(|(bytes, _)| bytes)
6391}
6392
6393fn task_bundle_is_absent(storage_dir: &Path, session_id: &str, task_id: &str) -> bool {
6394    let session_dir = session_tasks_dir(storage_dir, session_id);
6395    !session_dir.join(task_id).exists() && !session_dir.join(format!("{task_id}.json")).exists()
6396}
6397
6398fn terminal_db_row_snapshot(row: BashTaskRow, metadata: PersistedTask) -> BgTaskSnapshot {
6399    let existing_path = |path: Option<String>| {
6400        path.filter(|path| {
6401            fs::metadata(path)
6402                .map(|metadata| metadata.is_file())
6403                .unwrap_or(false)
6404        })
6405    };
6406    let duration_ms = metadata.duration_ms.or_else(|| {
6407        metadata
6408            .finished_at
6409            .map(|finished_at| finished_at.saturating_sub(metadata.started_at))
6410    });
6411    let live_descendants_summary = live_descendants_summary(&metadata);
6412    BgTaskSnapshot {
6413        info: BgTaskInfo {
6414            task_id: metadata.task_id,
6415            status: metadata.status,
6416            command: metadata.command,
6417            mode: metadata.mode.clone(),
6418            started_at: metadata.started_at,
6419            duration_ms,
6420            status_reason: metadata.status_reason,
6421        },
6422        exit_code: metadata.exit_code,
6423        child_pid: metadata.child_pid,
6424        workdir: metadata.workdir.display().to_string(),
6425        output_preview: String::new(),
6426        bash_output_list_envelope: None,
6427        output_truncated: false,
6428        output_path: existing_path(row.stdout_path),
6429        stderr_path: existing_path(row.stderr_path),
6430        pty_rows: (metadata.mode == BgMode::Pty).then_some(metadata.pty_rows.unwrap_or(24)),
6431        pty_cols: (metadata.mode == BgMode::Pty).then_some(metadata.pty_cols.unwrap_or(80)),
6432        pty_screen: None,
6433        scanner_report: metadata.scanner_report,
6434        sandbox_native: metadata.sandbox_native,
6435        sandbox_unavailable: false,
6436        live_descendants: metadata.live_descendants.clone(),
6437        live_descendants_omitted: metadata.live_descendants_omitted,
6438        live_descendants_summary,
6439        kill_signaled: false,
6440        kill_reached: 0,
6441    }
6442}
6443
6444impl BgTask {
6445    fn snapshot(&self, preview_bytes: usize) -> BgTaskSnapshot {
6446        let state = self
6447            .state
6448            .lock()
6449            .unwrap_or_else(|poison| poison.into_inner());
6450        self.snapshot_locked(&state, preview_bytes)
6451    }
6452
6453    fn snapshot_locked(&self, state: &BgTaskState, preview_bytes: usize) -> BgTaskSnapshot {
6454        let metadata = &state.metadata;
6455        let duration_ms = metadata.duration_ms.or_else(|| {
6456            metadata
6457                .status
6458                .is_terminal()
6459                .then(|| self.started.elapsed().as_millis() as u64)
6460        });
6461        let (output_preview, output_truncated) = if metadata.mode == BgMode::Pty {
6462            (String::new(), false)
6463        } else if metadata.status.is_terminal() {
6464            state
6465                .terminal_output_cache
6466                .as_ref()
6467                .map(|cache| (cache.output_preview.clone(), cache.output_truncated))
6468                .unwrap_or_else(|| (String::new(), false))
6469        } else if preview_bytes == 0 {
6470            (String::new(), false)
6471        } else {
6472            state.buffer.read_tail(preview_bytes)
6473        };
6474        BgTaskSnapshot {
6475            info: BgTaskInfo {
6476                task_id: self.task_id.clone(),
6477                status: metadata.status.clone(),
6478                command: metadata.command.clone(),
6479                mode: metadata.mode.clone(),
6480                started_at: metadata.started_at,
6481                duration_ms,
6482                status_reason: metadata.status_reason.clone(),
6483            },
6484            exit_code: metadata.exit_code,
6485            child_pid: metadata.child_pid,
6486            workdir: metadata.workdir.display().to_string(),
6487            output_preview,
6488            bash_output_list_envelope: None,
6489            output_truncated,
6490            output_path: state
6491                .buffer
6492                .output_path()
6493                .map(|path| path.display().to_string()),
6494            stderr_path: state
6495                .buffer
6496                .stderr_path()
6497                .map(|path| path.display().to_string()),
6498            pty_rows: (metadata.mode == BgMode::Pty).then_some(metadata.pty_rows.unwrap_or(24)),
6499            pty_cols: (metadata.mode == BgMode::Pty).then_some(metadata.pty_cols.unwrap_or(80)),
6500            pty_screen: None,
6501            scanner_report: metadata.scanner_report.clone(),
6502            sandbox_native: metadata.sandbox_native,
6503            sandbox_unavailable: metadata.sandbox_native
6504                && open_task_artifact(&self.paths, TaskArtifact::SandboxUnavailable)
6505                    .and_then(|mut file| file.read_all())
6506                    .is_ok_and(|bytes| bytes == b"sandbox_unavailable"),
6507            live_descendants: metadata.live_descendants.clone(),
6508            live_descendants_omitted: metadata.live_descendants_omitted,
6509            live_descendants_summary: live_descendants_summary(metadata),
6510            kill_signaled: false,
6511            kill_reached: 0,
6512        }
6513    }
6514
6515    pub(crate) fn is_running(&self) -> bool {
6516        self.state
6517            .lock()
6518            .map(|state| {
6519                state.metadata.status == BgTaskStatus::Running
6520                    || (state.metadata.mode == BgMode::Pty
6521                        && state.metadata.status == BgTaskStatus::Killing)
6522            })
6523            .unwrap_or(false)
6524    }
6525
6526    fn is_terminal(&self) -> bool {
6527        self.state
6528            .lock()
6529            .map(|state| state.metadata.status.is_terminal())
6530            .unwrap_or(false)
6531    }
6532
6533    fn mark_terminal_now(&self) {
6534        if let Ok(mut terminal_at) = self.terminal_at.lock() {
6535            if terminal_at.is_none() {
6536                *terminal_at = Some(Instant::now());
6537            }
6538        }
6539    }
6540
6541    fn set_completion_delivered(
6542        &self,
6543        delivered: bool,
6544        registry: &BgTaskRegistry,
6545    ) -> std::io::Result<()> {
6546        let mut state = self
6547            .state
6548            .lock()
6549            .map_err(|_| std::io::Error::other("background task lock poisoned"))?;
6550        let updated = registry.update_task_metadata(&self.paths, |metadata| {
6551            metadata.completion_delivered = delivered;
6552        })?;
6553        state.metadata = updated;
6554        Ok(())
6555    }
6556}
6557
6558/// Reap an exited direct child handle, then clear the slot.
6559///
6560/// Dropping a [`std::process::Child`] does NOT `wait()` on the underlying OS
6561/// process. On Unix a finished-but-unreaped child lingers as a `<defunct>`
6562/// zombie until the AFT process itself exits (issue #91: `[mv] <defunct>`).
6563/// The terminal-transition paths that learn of completion from the
6564/// exit-marker file — rather than from [`BgTaskRegistry::reap_child`]'s
6565/// `try_wait()` — must therefore reap the handle explicitly instead of just
6566/// nulling it.
6567///
6568/// The exit marker is written by the wrapper's final statement (an atomic
6569/// `mv` rename), so by the time we observe the marker the direct child has
6570/// finished its work and is exiting; `wait()` returns essentially
6571/// immediately. We attempt a non-blocking `try_wait()` first so the common
6572/// case never blocks at all, falling back to a (bounded) `wait()` only to
6573/// cover the microsecond window between the rename and process teardown.
6574///
6575/// Callers hold the task state mutex, so this is serialized against
6576/// `reap_child` — there is no double-`wait()` hazard: whichever path acquires
6577/// the lock first reaps and clears the slot, and the other observes `None`.
6578#[cfg(unix)]
6579fn reap_piped_child(child_slot: &mut Option<Child>) {
6580    if let Some(mut child) = child_slot.take() {
6581        if matches!(child.try_wait(), Ok(None)) {
6582            let _ = child.wait();
6583        }
6584    }
6585}
6586
6587/// Windows has no zombie/`<defunct>` concept: dropping the [`Child`] closes
6588/// the process handle, which is the correct release. Preserve the historical
6589/// behavior of simply clearing the slot so the documented Windows PID-recycle
6590/// handling in `reap_child` is unaffected.
6591#[cfg(windows)]
6592fn reap_piped_child(child_slot: &mut Option<Child>) {
6593    *child_slot = None;
6594}
6595
6596fn terminal_metadata_from_marker(
6597    mut metadata: PersistedTask,
6598    marker: ExitMarker,
6599    reason: Option<String>,
6600) -> PersistedTask {
6601    match marker {
6602        ExitMarker::Code(code) => {
6603            let status = if code == 0 {
6604                BgTaskStatus::Completed
6605            } else {
6606                BgTaskStatus::Failed
6607            };
6608            metadata.mark_terminal(status, Some(code), reason);
6609        }
6610        ExitMarker::Killed => metadata.mark_terminal(
6611            BgTaskStatus::Killed,
6612            terminal_exit_code_for_status(&BgTaskStatus::Killed),
6613            reason,
6614        ),
6615    }
6616    metadata
6617}
6618
6619fn terminal_exit_code_for_status(status: &BgTaskStatus) -> Option<i32> {
6620    match status {
6621        BgTaskStatus::TimedOut => Some(124),
6622        BgTaskStatus::Killed => Some(137),
6623        _ => None,
6624    }
6625}
6626
6627fn attach_sandbox_metadata(metadata: &mut PersistedTask, spawn_plan: &SpawnPlan) {
6628    metadata.sandbox_native = spawn_plan.is_native_launcher();
6629    metadata.sandbox_temp_dir = spawn_plan.temp_dir().map(Path::to_path_buf);
6630}
6631
6632#[cfg(unix)]
6633pub(crate) fn resolve_posix_shell() -> PathBuf {
6634    static POSIX_SHELL: OnceLock<PathBuf> = OnceLock::new();
6635    POSIX_SHELL
6636        .get_or_init(|| {
6637            std::env::var_os("BASH")
6638                .filter(|value| !value.is_empty())
6639                .map(PathBuf::from)
6640                .filter(|path| path.exists())
6641                .or_else(|| which::which("bash").ok())
6642                .or_else(|| which::which("zsh").ok())
6643                .unwrap_or_else(|| PathBuf::from("/bin/sh"))
6644        })
6645        .clone()
6646}
6647
6648#[cfg(windows)]
6649fn detached_shell_command_for(
6650    shell: crate::windows_shell::WindowsShell,
6651    command: &str,
6652    exit_path: &Path,
6653    paths: &TaskPaths,
6654    creation_flags: u32,
6655) -> Result<Command, String> {
6656    use crate::windows_shell::WindowsShell;
6657    // Write the wrapper to a temp file alongside the other task files,
6658    // then invoke the shell with the file path as a single clean
6659    // argument. This sidesteps the entire Windows command-line quoting
6660    // mess (Rust std-lib quoting + cmd /C parser + PowerShell -Command
6661    // parser all interacting with embedded quotes in the wrapper).
6662    //
6663    // Path arguments don't need quoting in the same problematic way
6664    // because: (1) we use no-space task IDs (bash-XXXXXXXX) so the path
6665    // contains no characters that need shell escaping; (2) the wrapper
6666    // body's internal quotes never reach the shell command line — the
6667    // shell reads them from disk by file syntax rules, not command-line
6668    // parser rules.
6669    let wrapper_body = shell.wrapper_script_bytes(command, exit_path);
6670    let wrapper_ext = match shell {
6671        WindowsShell::Pwsh | WindowsShell::Powershell => "ps1",
6672        WindowsShell::Cmd => "bat",
6673        // POSIX shells (git-bash etc.) execute the wrapper through `-c`,
6674        // so the file extension is purely cosmetic; `.sh` matches what an
6675        // operator would expect when grepping the spill directory.
6676        WindowsShell::Posix(_) => "sh",
6677    };
6678    let wrapper_path = paths.dir.join(format!(
6679        "{}.{}",
6680        paths
6681            .json
6682            .file_stem()
6683            .and_then(|s| s.to_str())
6684            .unwrap_or("wrapper"),
6685        wrapper_ext
6686    ));
6687    fs::write(&wrapper_path, wrapper_body)
6688        .map_err(|e| format!("failed to write background bash wrapper script: {e}"))?;
6689
6690    let mut cmd = Command::new(shell.binary().as_ref());
6691    match shell {
6692        WindowsShell::Pwsh | WindowsShell::Powershell => {
6693            // -File runs the script with no quoting issues. `-NoLogo`,
6694            // `-NoProfile`, etc. apply to the host before the file runs.
6695            cmd.args([
6696                "-NoLogo",
6697                "-NoProfile",
6698                "-NonInteractive",
6699                "-ExecutionPolicy",
6700                "Bypass",
6701                "-File",
6702            ]);
6703            cmd.arg(&wrapper_path);
6704        }
6705        WindowsShell::Cmd => {
6706            // `cmd /D /C "<bat-file-path>"` — invoking a .bat
6707            // file via /C is well-defined; the file's contents are
6708            // read line-by-line by cmd's batch processor, NOT
6709            // re-interpreted by the /C parser. This avoids the
6710            // "filename syntax incorrect" errors that came from
6711            // having complex compound commands on the cmd line.
6712            cmd.args(["/D", "/C"]);
6713            cmd.arg(&wrapper_path);
6714        }
6715        WindowsShell::Posix(_) => {
6716            // git-bash and other POSIX shells run the wrapper script with
6717            // `<binary> <wrapper-path>` (the wrapper is just a shell
6718            // script). No special flags needed — the `trap` and atomic
6719            // exit-marker rename in `wrapper_script` are POSIX-standard.
6720            cmd.arg(&wrapper_path);
6721        }
6722    }
6723
6724    // Win32 process creation flags. Caller selects whether to include
6725    // CREATE_BREAKAWAY_FROM_JOB — see `detached_shell_command_for` callers
6726    // for the breakaway-fallback strategy.
6727    cmd.creation_flags(creation_flags);
6728    Ok(cmd)
6729}
6730
6731/// Spawn a detached background bash child process.
6732///
6733/// On Unix this is a single spawn against `/bin/sh`. On Windows it walks
6734/// `WindowsShell::shell_candidates()` (pwsh.exe → powershell.exe →
6735/// cmd.exe) and retries with the next candidate when the previous one
6736/// fails to spawn with `NotFound` — the same runtime safety net the
6737/// foreground bash path has, so issue #27 callers landing on cmd.exe
6738/// fallback can also use background bash. The wrapper script is
6739/// regenerated per attempt because PowerShell wrappers embed the shell
6740/// binary by name; the stdout/stderr capture handles are also reopened
6741/// per attempt because `Command::spawn()` consumes them.
6742///
6743/// Errors other than `NotFound` (PermissionDenied, OutOfMemory, etc.)
6744/// return immediately without retry — they indicate a problem with the
6745/// resolved shell that retrying with a different shell won't fix.
6746fn spawn_detached_child(
6747    spawn_plan: &SpawnPlan,
6748    command: &str,
6749    shell: super::BashShell,
6750    shell_path: &Path,
6751    paths: &TaskPaths,
6752    workdir: &Path,
6753    env: &HashMap<String, String>,
6754    io_handles: &mut TaskIoHandles,
6755    capture_pipeline_status: bool,
6756    #[cfg_attr(not(target_os = "linux"), allow(unused_variables))] linux_scope: bool,
6757) -> Result<std::process::Child, String> {
6758    #[cfg(windows)]
6759    let _ = capture_pipeline_status;
6760    #[cfg(not(windows))]
6761    let _ = (command, shell);
6762    #[cfg(not(windows))]
6763    {
6764        use std::os::fd::AsRawFd;
6765
6766        let stdout = io_handles
6767            .clone_file(TaskArtifact::Stdout)
6768            .map_err(|e| format!("failed to clone stdout capture handle: {e}"))?;
6769        let stderr = io_handles
6770            .clone_file(TaskArtifact::Stderr)
6771            .map_err(|e| format!("failed to clone stderr capture handle: {e}"))?;
6772        let prepared = spawn_plan
6773            .prepared_task()
6774            .ok_or_else(|| "background task payload was not prepared".to_string())?;
6775        let payload = prepared.invocation()?;
6776        let exit = io_handles
6777            .inheritable_file(TaskArtifact::Exit)
6778            .map_err(|e| format!("failed to inherit exit marker handle: {e}"))?;
6779        let failure = io_handles
6780            .inheritable_file(TaskArtifact::SandboxUnavailable)
6781            .map_err(|e| format!("failed to inherit sandbox failure marker handle: {e}"))?;
6782        let pipeline_status = capture_pipeline_status
6783            .then(|| io_handles.inheritable_file(TaskArtifact::PipelineStatus))
6784            .transpose()
6785            .map_err(|e| format!("failed to inherit pipeline status handle: {e}"))?;
6786        let shell_path = spawn_plan.host_shell_path().unwrap_or(shell_path);
6787        let pipeline_shell = super::process::pipeline_shell_kind(shell_path).unwrap_or("");
6788        let pipeline_status_fd = if capture_pipeline_status {
6789            crate::sandbox_spawn::CHILD_PIPE_STATUS_FD.to_string()
6790        } else {
6791            String::new()
6792        };
6793        let base_executable = PathBuf::from("/bin/sh");
6794        let base_args = vec![
6795            OsString::from("-c"),
6796            payload.wrapper_text.clone(),
6797            OsString::from("aft-payload-wrapper"),
6798            shell_path.as_os_str().to_os_string(),
6799            payload.command_text.clone(),
6800            OsString::from(crate::sandbox_spawn::CHILD_EXIT_FD.to_string()),
6801            OsString::from(pipeline_status_fd),
6802            OsString::from(pipeline_shell),
6803        ];
6804        #[cfg(target_os = "linux")]
6805        let (executable, args) = if linux_scope {
6806            if spawn_plan.is_native_launcher() {
6807                crate::slog_info!(
6808                    "bash.linux_scope requested but native sandbox launch cannot reach the user manager; spawning without a systemd scope"
6809                );
6810                (base_executable, base_args)
6811            } else if let Some(launcher) = super::process::select_systemd_scope_launcher() {
6812                super::process::systemd_scope_argv(&launcher, &base_executable, &base_args)
6813            } else {
6814                crate::slog_info!(
6815                    "bash.linux_scope requested but systemd-run or the user manager is unavailable; spawning without a systemd scope"
6816                );
6817                (base_executable, base_args)
6818            }
6819        } else {
6820            (base_executable, base_args)
6821        };
6822        #[cfg(not(target_os = "linux"))]
6823        let (executable, args) = {
6824            let _ = linux_scope;
6825            (base_executable, base_args)
6826        };
6827        let (mut child_command, profile_handle) = crate::sandbox_spawn::detached_command_for_plan(
6828            spawn_plan,
6829            executable.as_os_str(),
6830            &args,
6831            &paths.json,
6832            crate::sandbox_spawn::CHILD_EXIT_FD,
6833            crate::sandbox_spawn::CHILD_FAILURE_FD,
6834        )?;
6835        crate::sandbox_spawn::apply_marker_fd_allowlist(
6836            &mut child_command,
6837            exit.as_raw_fd(),
6838            failure.as_raw_fd(),
6839            pipeline_status.as_ref().map(|file| file.as_raw_fd()),
6840        )?;
6841        child_command
6842            .current_dir(workdir)
6843            .stdin(Stdio::null())
6844            .stdout(Stdio::from(stdout))
6845            .stderr(Stdio::from(stderr));
6846        crate::agent_child_env::apply_to_command(&mut child_command, env);
6847        crate::sandbox_spawn::apply_sandbox_environment(spawn_plan, &mut child_command, env);
6848        let child = child_command
6849            .spawn()
6850            .map_err(|e| format!("failed to spawn background bash command: {e}"));
6851        drop((payload, exit, failure, pipeline_status, profile_handle));
6852        child
6853    }
6854    #[cfg(windows)]
6855    {
6856        let _ = shell_path;
6857        use crate::windows_shell::shell_candidates;
6858        match spawn_plan {
6859            SpawnPlan::Unsandboxed | SpawnPlan::Host { .. } => {}
6860            SpawnPlan::Refused { code, .. } => return Err((*code).to_string()),
6861            SpawnPlan::Launcher { .. } => return Err("sandbox_unavailable".to_string()),
6862        }
6863        // Spawn priority: pwsh → powershell → git-bash → cmd. Same as the
6864        // legacy foreground bash spawn path. v0.20 routes ALL bash through
6865        // this background spawn helper, including foreground tool calls
6866        // where the model writes PowerShell-syntax (`$var = ...`,
6867        // `Start-Sleep`, `Add-Content`) — those fail outright under cmd.
6868        // The earlier v0.18-era cmd-first override worked around a
6869        // PowerShell detached-output bug; that bug is fixed at the
6870        // process-flag layer (CREATE_NO_WINDOW instead of DETACHED_PROCESS,
6871        // see flag block below), so we no longer need to misroute PS
6872        // commands through cmd.
6873        let candidates: Vec<crate::windows_shell::WindowsShell> = if shell.is_powershell() {
6874            vec![crate::windows_shell::WindowsShell::Pwsh]
6875        } else {
6876            shell_candidates()
6877        };
6878        // Win32 process creation flags. We try with CREATE_BREAKAWAY_FROM_JOB
6879        // first (so the bg child outlives the AFT process when AFT is killed),
6880        // then fall back without it for environments where the parent is in a
6881        // Job Object that doesn't grant `JOB_OBJECT_LIMIT_BREAKAWAY_OK`. CI
6882        // runners (GitHub Actions windows-2022) and some MDM-managed corp
6883        // environments hit this — `CreateProcess` returns Access Denied (5).
6884        // Without breakaway, the child still runs detached but will be torn
6885        // down with the parent if the parent process group is signaled.
6886        //
6887        // CREATE_NO_WINDOW avoids a visible console while retaining the
6888        // hidden console services PowerShell needs for reliable redirected
6889        // stdout/stderr. DETACHED_PROCESS can drop redirected output.
6890        const FLAG_CREATE_NEW_PROCESS_GROUP: u32 = 0x0000_0200;
6891        const FLAG_CREATE_BREAKAWAY_FROM_JOB: u32 = 0x0100_0000;
6892        const FLAG_CREATE_NO_WINDOW: u32 = 0x0800_0000;
6893        let with_breakaway =
6894            FLAG_CREATE_NO_WINDOW | FLAG_CREATE_NEW_PROCESS_GROUP | FLAG_CREATE_BREAKAWAY_FROM_JOB;
6895        let without_breakaway = FLAG_CREATE_NO_WINDOW | FLAG_CREATE_NEW_PROCESS_GROUP;
6896        let mut last_error: Option<String> = None;
6897        for (idx, shell) in candidates.iter().enumerate() {
6898            // Per-shell, try with breakaway first. If the process is in a
6899            // restrictive job, the breakaway flag triggers Access Denied
6900            // (os error 5). Retry once without breakaway.
6901            for &flags in &[with_breakaway, without_breakaway] {
6902                // Clone the pre-opened O_EXCL capture handles per attempt;
6903                // Command::spawn consumes each Stdio wrapper.
6904                let stdout = io_handles
6905                    .clone_file(TaskArtifact::Stdout)
6906                    .map_err(|e| format!("failed to clone stdout capture handle: {e}"))?;
6907                let stderr = io_handles
6908                    .clone_file(TaskArtifact::Stderr)
6909                    .map_err(|e| format!("failed to clone stderr capture handle: {e}"))?;
6910                let mut cmd =
6911                    detached_shell_command_for(shell.clone(), command, &paths.exit, paths, flags)?;
6912                cmd.current_dir(workdir)
6913                    .stdin(Stdio::null())
6914                    .stdout(Stdio::from(stdout))
6915                    .stderr(Stdio::from(stderr));
6916                crate::agent_child_env::apply_to_command(&mut cmd, env);
6917                match cmd.spawn() {
6918                    Ok(child) => {
6919                        if idx > 0 {
6920                            crate::slog_warn!("background bash spawn fell back to {} after {} earlier candidate(s) failed; \
6921                             the cached PATH probe disagreed with runtime spawn — likely PATH \
6922                             inheritance, antivirus / AppLocker / Defender ASR, or sandbox policy.",
6923                            shell.binary(),
6924                            idx);
6925                        }
6926                        if flags == without_breakaway {
6927                            crate::slog_warn!(
6928                                "background bash spawn: CREATE_BREAKAWAY_FROM_JOB rejected \
6929                             (likely a restrictive Job Object — CI sandbox or MDM policy). \
6930                             Spawned without breakaway; the bg task will be torn down if the \
6931                             AFT process group is killed."
6932                            );
6933                        }
6934                        return Ok(child);
6935                    }
6936                    Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
6937                        crate::slog_warn!("background bash spawn: {} returned NotFound at runtime — trying next candidate",
6938                        shell.binary());
6939                        last_error = Some(format!("{}: {e}", shell.binary()));
6940                        // Skip the without-breakaway retry for NotFound — the
6941                        // binary itself is missing, breakaway flag is irrelevant.
6942                        break;
6943                    }
6944                    Err(e) if flags == with_breakaway && e.raw_os_error() == Some(5) => {
6945                        // Access Denied during breakaway — retry without it.
6946                        crate::slog_warn!(
6947                            "background bash spawn: CREATE_BREAKAWAY_FROM_JOB rejected with \
6948                         Access Denied — retrying {} without breakaway",
6949                            shell.binary()
6950                        );
6951                        last_error = Some(format!("{}: {e}", shell.binary()));
6952                        continue;
6953                    }
6954                    Err(e) => {
6955                        return Err(format!(
6956                            "failed to spawn background bash command via {}: {e}",
6957                            shell.binary()
6958                        ));
6959                    }
6960                }
6961            }
6962        }
6963        Err(format!(
6964            "failed to spawn background bash command: no Windows shell could be spawned. \
6965             Last error: {}. PATH-probed candidates: {:?}",
6966            last_error.unwrap_or_else(|| "no candidates were attempted".to_string()),
6967            candidates.iter().map(|s| s.binary()).collect::<Vec<_>>()
6968        ))
6969    }
6970}
6971
6972#[cfg(test)]
6973fn random_slug() -> String {
6974    // 8 bytes = 64-bit entropy → `bash-{16hex}`, matching the documented contract
6975    // at `generate_unique_task_id`. The width is load-bearing for the subc
6976    // delivery dedup: a plugin can retain a delivered task id awaiting ack that
6977    // Rust has already dropped (a lost ack response), and Rust's uniqueness check
6978    // cannot see that plugin-side set — so id reuse must be made negligible by
6979    // entropy alone. 32-bit was reusable within a long session and could let a new
6980    // task collide with such a stale id and be silently skipped (audit R3 #3).
6981    let mut bytes = [0u8; 8];
6982    // getrandom is a transitive dependency; use it directly for OS entropy.
6983    getrandom::fill(&mut bytes).unwrap_or_else(|_| {
6984        // Extremely unlikely fallback: time + pid mix across all 8 bytes.
6985        let t = SystemTime::now()
6986            .duration_since(UNIX_EPOCH)
6987            .map(|d| d.as_nanos() as u64)
6988            .unwrap_or(0);
6989        let p = u64::from(std::process::id());
6990        bytes.copy_from_slice(&(t ^ p.rotate_left(32)).to_le_bytes());
6991    });
6992    // `bash-` + 16 lowercase hex chars — compact, OS-entropy backed.
6993    let hex: String = bytes.iter().map(|b| format!("{b:02x}")).collect();
6994    format!("bash-{hex}")
6995}
6996
6997#[cfg(test)]
6998mod tests {
6999    use std::collections::HashMap;
7000    use std::fs;
7001    use std::io::Write;
7002    #[cfg(unix)]
7003    use std::os::unix::fs::PermissionsExt;
7004    use std::sync::atomic::{AtomicBool, AtomicUsize};
7005    use std::sync::{Arc, Mutex};
7006    use std::time::{Duration, Instant, SystemTime};
7007
7008    use super::*;
7009    use crate::bash_background::persistence::{read_task, task_paths, write_task};
7010
7011    #[cfg(unix)]
7012    const QUICK_SUCCESS_COMMAND: &str = "true";
7013    #[cfg(windows)]
7014    const QUICK_SUCCESS_COMMAND: &str = "cmd /c exit 0";
7015
7016    /// Upper bound for "a trivial child terminates". This asserts liveness, not
7017    /// speed: under a parallel libtest run on a contended Windows CI runner a
7018    /// `cmd /c exit 0` spawn has taken longer than the 5s these tests once
7019    /// allowed, and the scheduler's delay is not the product's.
7020    const CHILD_EXIT_LIVENESS_BOUND: Duration = Duration::from_secs(30);
7021
7022    #[cfg(unix)]
7023    const LONG_RUNNING_COMMAND: &str = "sleep 5";
7024
7025    #[cfg(unix)]
7026    #[test]
7027    fn launcher_plans_disable_pipeline_status_capture() {
7028        let launcher = SpawnPlan::launcher_for_test(
7029            crate::sandbox_profile::SandboxProfile {
7030                v: crate::sandbox_profile::SANDBOX_PROFILE_VERSION,
7031                writable_roots: Vec::new(),
7032                write_deny: Vec::new(),
7033                write_deny_nested: Vec::new(),
7034                read_allow: Vec::new(),
7035                read_deny: Vec::new(),
7036                socket_deny: Vec::new(),
7037                cache_roots: Vec::new(),
7038                temp_dir: PathBuf::from("/tmp/aft-test-sandbox"),
7039            },
7040            PathBuf::from("/bin/true"),
7041        );
7042        assert!(!should_capture_pipeline_status(
7043            &launcher,
7044            true,
7045            Path::new("/bin/bash")
7046        ));
7047        assert!(should_capture_pipeline_status(
7048            &SpawnPlan::Unsandboxed,
7049            true,
7050            Path::new("/bin/bash")
7051        ));
7052    }
7053
7054    #[cfg(windows)]
7055    const LONG_RUNNING_COMMAND: &str = "cmd /c timeout /t 5 /nobreak > nul";
7056
7057    #[test]
7058    fn bash_memory_estimate_is_zero_when_empty_and_nonzero_for_completion_cache() {
7059        let registry = BgTaskRegistry::default();
7060        assert_eq!(registry.estimated_memory().estimated_bytes, Some(0));
7061        registry
7062            .inner
7063            .completions
7064            .lock()
7065            .unwrap()
7066            .push_back(BgCompletion {
7067                task_id: "bash-memory".to_string(),
7068                session_id: "session-memory".to_string(),
7069                status: BgTaskStatus::Completed,
7070                exit_code: Some(0),
7071                command: "printf memory".to_string(),
7072                output_preview: "resident completion output".to_string(),
7073                bash_output_list_envelope: None,
7074                output_truncated: false,
7075                original_tokens: None,
7076                compressed_tokens: None,
7077                tokens_skipped: false,
7078                status_reason: None,
7079                live_descendants: Some(Vec::new()),
7080                live_descendants_omitted: 0,
7081                live_descendants_summary: None,
7082            });
7083        let estimate = registry.estimated_memory();
7084        assert!(estimate.estimated_bytes.unwrap() > 0);
7085        assert_eq!(estimate.counts["completion_caches"], 1);
7086        assert_eq!(estimate.counts["sessions"], 1);
7087    }
7088
7089    #[test]
7090    fn gh_structured_detection_rejects_piped_commands() {
7091        assert!(is_gh_structured_command(
7092            "gh issue list --json number,title"
7093        ));
7094        assert!(is_gh_structured_command(
7095            "cd repo && gh issue list --json number,title"
7096        ));
7097
7098        assert!(!is_gh_structured_command(
7099            "gh issue list --json number,title | jq '.[]'"
7100        ));
7101        assert!(!is_gh_structured_command(
7102            "gh issue list --json number,title |"
7103        ));
7104    }
7105
7106    fn insert_terminal_piped_task(
7107        registry: &BgTaskRegistry,
7108        dir: &tempfile::TempDir,
7109        command: &str,
7110        stdout: &str,
7111        stderr: &str,
7112        compressed: bool,
7113    ) -> (String, Arc<BgTask>) {
7114        let task_id = random_slug();
7115        let paths = task_paths(dir.path(), "session", &task_id).unwrap();
7116        fs::create_dir_all(&paths.dir).unwrap();
7117        fs::write(&paths.stdout, stdout).unwrap();
7118        fs::write(&paths.stderr, stderr).unwrap();
7119        let mut metadata = PersistedTask::starting(
7120            task_id.clone(),
7121            "session".to_string(),
7122            command.to_string(),
7123            dir.path().to_path_buf(),
7124            Some(dir.path().to_path_buf()),
7125            Some(30_000),
7126            true,
7127            compressed,
7128        );
7129        metadata.mark_terminal(BgTaskStatus::Completed, Some(0), None);
7130        write_task(&paths.json, &metadata).unwrap();
7131        registry
7132            .insert_rehydrated_task(metadata, paths, true)
7133            .expect("insert terminal task");
7134        let task = registry.task_for_session(&task_id, "session").unwrap();
7135        (task_id, task)
7136    }
7137
7138    #[test]
7139    fn bash_zero_preview_running_status_skips_output_read_while_explicit_preview_reads() {
7140        let registry = BgTaskRegistry::default();
7141        let dir = tempfile::tempdir().unwrap();
7142        let task_id = random_slug();
7143        let paths = task_paths(dir.path(), "session", &task_id).unwrap();
7144        fs::create_dir_all(&paths.dir).unwrap();
7145        fs::write(&paths.stdout, "live output\n").unwrap();
7146        fs::write(&paths.stderr, "").unwrap();
7147        let stdout_path = paths.stdout.clone();
7148        let mut metadata = PersistedTask::starting(
7149            task_id.clone(),
7150            "session".to_string(),
7151            "sleep 60".to_string(),
7152            dir.path().to_path_buf(),
7153            Some(dir.path().to_path_buf()),
7154            Some(30_000),
7155            true,
7156            false,
7157        );
7158        metadata.status = BgTaskStatus::Running;
7159        write_task(&paths.json, &metadata).unwrap();
7160        registry
7161            .insert_rehydrated_task(metadata, paths, false)
7162            .expect("insert running task");
7163
7164        crate::bash_background::buffer::reset_tail_read_count(&stdout_path);
7165        for _ in 0..5 {
7166            let snapshot = registry
7167                .status(&task_id, "session", Some(dir.path()), Some(dir.path()), 0)
7168                .expect("running snapshot");
7169            assert_eq!(snapshot.info.status, BgTaskStatus::Running);
7170            assert!(snapshot.output_preview.is_empty());
7171        }
7172        assert_eq!(
7173            crate::bash_background::buffer::tail_read_count(&stdout_path),
7174            0
7175        );
7176
7177        let snapshot = registry
7178            .status(
7179                &task_id,
7180                "session",
7181                Some(dir.path()),
7182                Some(dir.path()),
7183                RUNNING_OUTPUT_PREVIEW_BYTES,
7184            )
7185            .expect("explicit running snapshot");
7186        assert_eq!(snapshot.output_preview, "live output\n");
7187        assert_eq!(
7188            crate::bash_background::buffer::tail_read_count(&stdout_path),
7189            1
7190        );
7191    }
7192
7193    #[test]
7194    fn artifact_read_capability_requires_exact_canonical_path_and_session() {
7195        let registry = BgTaskRegistry::default();
7196        let dir = tempfile::tempdir().unwrap();
7197        let (_task_id, task) = insert_terminal_piped_task(
7198            &registry,
7199            &dir,
7200            "printf output",
7201            "stdout\n",
7202            "stderr\n",
7203            true,
7204        );
7205        fs::write(&task.paths.exit, "0\n").unwrap();
7206
7207        assert!(registry.is_session_owned_artifact_path("session", &task.paths.stdout));
7208        assert!(registry.is_session_owned_artifact_path("session", &task.paths.stderr));
7209        assert!(registry.is_session_owned_artifact_path("session", &task.paths.exit));
7210        assert!(!registry.is_session_owned_artifact_path("different-session", &task.paths.stdout));
7211        assert!(!registry.is_session_owned_artifact_path("session", &task.paths.json));
7212
7213        let unregistered = task.paths.dir.join("unregistered-output");
7214        fs::write(&unregistered, "not a task artifact\n").unwrap();
7215        assert!(!registry.is_session_owned_artifact_path("session", &unregistered));
7216    }
7217
7218    #[cfg(unix)]
7219    #[test]
7220    fn artifact_directory_symlink_does_not_create_a_prefix_exception() {
7221        let registry = BgTaskRegistry::default();
7222        let dir = tempfile::tempdir().unwrap();
7223        let project = dir.path().join("project");
7224        fs::create_dir_all(&project).unwrap();
7225        let (_task_id, task) =
7226            insert_terminal_piped_task(&registry, &dir, "printf output", "stdout\n", "", true);
7227        let link = project.join("task-artifacts");
7228        std::os::unix::fs::symlink(&task.paths.dir, &link).unwrap();
7229        let unregistered = task.paths.dir.join("unregistered-output");
7230        fs::write(&unregistered, "not registered\n").unwrap();
7231
7232        assert!(!registry.is_session_owned_artifact_path("session", &link));
7233        assert!(
7234            !registry.is_session_owned_artifact_path("session", &link.join("unregistered-output"))
7235        );
7236        assert!(registry.is_session_owned_artifact_path(
7237            "session",
7238            &link.join(task.paths.stdout.file_name().unwrap())
7239        ));
7240
7241        let outside = dir.path().join("outside-secret");
7242        fs::write(&outside, "must stay private\n").unwrap();
7243        fs::remove_file(&task.paths.stdout).unwrap();
7244        std::os::unix::fs::symlink(&outside, &task.paths.stdout).unwrap();
7245        assert!(!registry.is_session_owned_artifact_path("session", &task.paths.stdout));
7246    }
7247
7248    #[test]
7249    fn recovery_footer_uses_bash_status_when_artifact_is_not_registered() {
7250        let registry = BgTaskRegistry::default();
7251        let dir = tempfile::tempdir().unwrap();
7252        let task_id = "bash-1111111111111111";
7253        let paths = task_paths(dir.path(), "session", task_id).unwrap();
7254        fs::create_dir_all(&paths.dir).unwrap();
7255        fs::write(
7256            &paths.stdout,
7257            format!("{}tail\n", "output-line\n".repeat(2_000)),
7258        )
7259        .unwrap();
7260        fs::write(&paths.stderr, "").unwrap();
7261        let mut metadata = PersistedTask::starting(
7262            task_id.to_string(),
7263            "session".to_string(),
7264            "printf output".to_string(),
7265            dir.path().to_path_buf(),
7266            Some(dir.path().to_path_buf()),
7267            Some(30_000),
7268            true,
7269            true,
7270        );
7271        metadata.mark_terminal(BgTaskStatus::Completed, Some(0), None);
7272        write_task(&paths.json, &metadata).unwrap();
7273
7274        let cache = registry
7275            .render_terminal_output_from_paths(&metadata, &paths)
7276            .expect("terminal render");
7277
7278        assert!(cache
7279            .output_preview
7280            .contains("use bash_status({taskId: \"bash-1111111111111111\"})"));
7281        assert!(!cache.output_preview.contains("full output: read "));
7282    }
7283
7284    fn insert_terminal_pty_task(
7285        registry: &BgTaskRegistry,
7286        dir: &tempfile::TempDir,
7287        pty_output: &str,
7288    ) -> (String, Arc<BgTask>) {
7289        let task_id = random_slug();
7290        let paths = task_paths(dir.path(), "session", &task_id).unwrap();
7291        fs::create_dir_all(&paths.dir).unwrap();
7292        fs::write(&paths.pty, pty_output).unwrap();
7293        let mut metadata = PersistedTask::starting(
7294            task_id.clone(),
7295            "session".to_string(),
7296            "python".to_string(),
7297            dir.path().to_path_buf(),
7298            Some(dir.path().to_path_buf()),
7299            Some(30_000),
7300            true,
7301            true,
7302        );
7303        metadata.mode = BgMode::Pty;
7304        metadata.mark_terminal(BgTaskStatus::Completed, Some(0), None);
7305        write_task(&paths.json, &metadata).unwrap();
7306        registry
7307            .insert_rehydrated_task(metadata, paths, true)
7308            .expect("insert terminal pty task");
7309        let task = registry.task_for_session(&task_id, "session").unwrap();
7310        (task_id, task)
7311    }
7312
7313    #[cfg(unix)]
7314    fn wait_for_terminal_snapshot(
7315        registry: &BgTaskRegistry,
7316        task_id: &str,
7317        session_id: &str,
7318        project: &Path,
7319        storage: &Path,
7320    ) -> BgTaskSnapshot {
7321        let started = Instant::now();
7322        loop {
7323            let snapshot = registry
7324                .status(task_id, session_id, Some(project), Some(storage), 4096)
7325                .expect("spawned task should be visible to status");
7326            if snapshot.info.status.is_terminal() {
7327                return snapshot;
7328            }
7329            assert!(
7330                started.elapsed() < Duration::from_secs(10),
7331                "timed out waiting for task {task_id} to finish; last status={:?}",
7332                snapshot.info.status
7333            );
7334            std::thread::sleep(Duration::from_millis(50));
7335        }
7336    }
7337
7338    fn write_running_project_task(storage: &Path, project: &Path, session: &str, task_id: &str) {
7339        let paths = task_paths(storage, session, task_id).unwrap();
7340        let mut metadata = PersistedTask::starting(
7341            task_id.to_string(),
7342            session.to_string(),
7343            "sleep 60".to_string(),
7344            project.to_path_buf(),
7345            Some(project.to_path_buf()),
7346            Some(30_000),
7347            true,
7348            true,
7349        );
7350        metadata.status = BgTaskStatus::Running;
7351        // The harness's own PID is an always-alive process for READ-ONLY
7352        // paths (status replay never signals it). Kill-path tests must never
7353        // copy this: recording the harness PID where the product kills
7354        // child_pid takes down the whole libtest process on Windows - spawn
7355        // a disposable child instead (see bash_kill.rs tests).
7356        metadata.child_pid = Some(std::process::id());
7357        write_task(&paths.json, &metadata).unwrap();
7358        fs::write(&paths.stdout, "still running\n").unwrap();
7359        fs::write(&paths.stderr, "").unwrap();
7360    }
7361
7362    #[test]
7363    fn status_replay_filters_same_session_by_project_root() {
7364        let project_a = tempfile::tempdir().unwrap();
7365        let project_b = tempfile::tempdir().unwrap();
7366        let storage = tempfile::tempdir().unwrap();
7367        let session = "shared-session";
7368        let task_id = "bash-2222222222222222";
7369        write_running_project_task(storage.path(), project_a.path(), session, task_id);
7370
7371        let actor_b = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
7372        assert!(actor_b
7373            .status(
7374                task_id,
7375                session,
7376                Some(project_b.path()),
7377                Some(storage.path()),
7378                1024,
7379            )
7380            .is_none());
7381        assert!(actor_b.task_for_session(task_id, session).is_none());
7382
7383        let actor_a = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
7384        let snapshot = actor_a
7385            .status(
7386                task_id,
7387                session,
7388                Some(project_a.path()),
7389                Some(storage.path()),
7390                1024,
7391            )
7392            .expect("owning project should replay its task");
7393        assert_eq!(snapshot.info.status, BgTaskStatus::Running);
7394    }
7395
7396    #[cfg(unix)]
7397    #[test]
7398    fn multiline_pipeline_stdout_persists_all_lines_after_terminal_status() {
7399        let cases = [
7400            (
7401                "long-first",
7402                "sleep 0.5; printf 'one\\n' | cat\nprintf 'two\\n' | grep -c two\nprintf 'three\\n' | cat",
7403                vec!["one", "1", "three"],
7404            ),
7405            (
7406                "short-first",
7407                "printf 'one\\n' | cat\nsleep 0.2; printf 'two\\n' | grep -c two\nprintf 'three\\n' | cat",
7408                vec!["one", "1", "three"],
7409            ),
7410            (
7411                "failing-middle",
7412                "sleep 0.2; printf 'one\\n' | cat\nfalse; printf 'after-false\\n' | cat\nprintf 'three\\n' | cat",
7413                vec!["one", "after-false", "three"],
7414            ),
7415        ];
7416
7417        for (name, command, expected_lines) in cases {
7418            let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
7419            let dir = tempfile::tempdir().unwrap();
7420            let session_id = format!("session-{name}");
7421            let task_id = registry
7422                .spawn(
7423                    SpawnPlan::Unsandboxed,
7424                    command,
7425                    session_id.clone(),
7426                    dir.path().to_path_buf(),
7427                    HashMap::new(),
7428                    Some(Duration::from_secs(30)),
7429                    dir.path().to_path_buf(),
7430                    10,
7431                    true,
7432                    true,
7433                    Some(dir.path().to_path_buf()),
7434                )
7435                .unwrap();
7436
7437            let snapshot = wait_for_terminal_snapshot(
7438                &registry,
7439                &task_id,
7440                &session_id,
7441                dir.path(),
7442                dir.path(),
7443            );
7444            assert_eq!(
7445                snapshot.info.status,
7446                BgTaskStatus::Completed,
7447                "{name}: task should complete; snapshot={snapshot:?}"
7448            );
7449            assert_eq!(
7450                snapshot.exit_code,
7451                Some(0),
7452                "{name}: script should use the final command's exit code"
7453            );
7454
7455            let stdout = String::from_utf8(
7456                registry
7457                    .read_artifact(&task_id, &session_id, TaskArtifact::Stdout)
7458                    .expect("read validated stdout artifact"),
7459            )
7460            .expect("stdout is UTF-8");
7461            let lines: Vec<&str> = stdout.lines().collect();
7462            assert_eq!(
7463                lines, expected_lines,
7464                "{name}: raw stdout artifact must include every newline-separated command's output"
7465            );
7466        }
7467    }
7468
7469    #[test]
7470    fn recognizes_all_recovery_marker_forms() {
7471        assert!(is_recovery_marker(
7472            "[truncated output; full output: read \"/tmp/out\"]"
7473        ));
7474        assert!(is_recovery_marker(
7475            "[omitted output; see remaining: tail -n +42 \"/tmp/out\"]"
7476        ));
7477        assert!(is_recovery_marker(
7478            "[truncated output; full output unavailable]"
7479        ));
7480        assert!(is_recovery_marker(
7481            r#"[truncated 123 bytes from saved output prefix; retained output: read "/tmp/out"]"#
7482        ));
7483    }
7484
7485    #[test]
7486    fn recovery_marker_reports_disk_prefix_truncation_as_retained_output() {
7487        let recovery = RecoveryContext {
7488            dropped_by_class: BTreeMap::new(),
7489            had_inner_drop: false,
7490            offset_hint_eligible: false,
7491            offset_start_line: None,
7492            byte_truncated: false,
7493            disk_truncated_prefix_bytes: 4096,
7494            output_path: Some("/tmp/stdout".to_string()),
7495            stderr_path: None,
7496            include_stderr_path: false,
7497            artifact_access: ArtifactRecoveryAccess {
7498                task_id: "bash-test".to_string(),
7499                readable: true,
7500            },
7501        };
7502
7503        let marker = recovery_marker(&recovery).expect("disk truncation must emit marker");
7504
7505        assert!(marker.contains("truncated 4096 bytes from saved output prefix"));
7506        assert!(marker.contains(r#"retained output: read "/tmp/stdout""#));
7507        assert!(!marker.contains("full output: read"));
7508    }
7509
7510    #[test]
7511    fn killed_exit_marker_sets_nonzero_sentinel_exit_code() {
7512        let metadata = PersistedTask::starting(
7513            "task".to_string(),
7514            "session".to_string(),
7515            "cargo test".to_string(),
7516            PathBuf::from("/tmp"),
7517            None,
7518            None,
7519            true,
7520            true,
7521        );
7522
7523        let terminal = terminal_metadata_from_marker(metadata, ExitMarker::Killed, None);
7524
7525        assert_eq!(terminal.status, BgTaskStatus::Killed);
7526        assert_eq!(terminal.exit_code, Some(137));
7527    }
7528
7529    #[test]
7530    fn terminal_status_polls_use_cached_render_once_and_off_lock() {
7531        let registry = BgTaskRegistry::default();
7532        let dir = tempfile::tempdir().unwrap();
7533        let (_task_id, task) = insert_terminal_piped_task(
7534            &registry,
7535            &dir,
7536            "custom-tool --verbose",
7537            &"stdout line\n".repeat(200_000),
7538            "",
7539            true,
7540        );
7541        let calls = Arc::new(AtomicUsize::new(0));
7542        let saw_unlocked_state = Arc::new(AtomicBool::new(false));
7543        let task_holder = Arc::new(Mutex::new(Some(Arc::clone(&task))));
7544        let calls_for_closure = Arc::clone(&calls);
7545        let unlocked_for_closure = Arc::clone(&saw_unlocked_state);
7546        let task_for_closure = Arc::clone(&task_holder);
7547        registry.set_compressor_with_exit_code(move |_command, output, _exit_code| {
7548            calls_for_closure.fetch_add(1, Ordering::SeqCst);
7549            if let Some(task) = task_for_closure.lock().unwrap().as_ref() {
7550                if task.state.try_lock().is_ok() {
7551                    unlocked_for_closure.store(true, Ordering::SeqCst);
7552                }
7553            }
7554            CompressionResult::new(format!("compressed {} bytes", output.len()))
7555        });
7556
7557        let first = registry
7558            .status(
7559                &task.task_id,
7560                "session",
7561                None,
7562                Some(dir.path()),
7563                RUNNING_OUTPUT_PREVIEW_BYTES,
7564            )
7565            .unwrap();
7566        let second = registry
7567            .status(
7568                &task.task_id,
7569                "session",
7570                None,
7571                Some(dir.path()),
7572                RUNNING_OUTPUT_PREVIEW_BYTES,
7573            )
7574            .unwrap();
7575        let listed = registry.list(RUNNING_OUTPUT_PREVIEW_BYTES);
7576
7577        assert_eq!(
7578            calls.load(Ordering::SeqCst),
7579            1,
7580            "terminal render must be cached"
7581        );
7582        assert!(
7583            saw_unlocked_state.load(Ordering::SeqCst),
7584            "compressor must run after releasing the task state lock"
7585        );
7586        assert!(first.output_preview.starts_with("compressed "));
7587        assert_eq!(second.output_preview, first.output_preview);
7588        assert_eq!(listed[0].output_preview, first.output_preview);
7589    }
7590
7591    #[test]
7592    fn completion_preview_success_keeps_tail_only() {
7593        // Exit-aware completion previews: a SUCCESSFUL task's reminder keeps a
7594        // short tail only — head context is noise when the command worked
7595        // (regression: the uniform 4 KiB head+tail cap flooded reminders with
7596        // ~1K tokens of build noise per completed task).
7597        let registry = BgTaskRegistry::default();
7598        let dir = tempfile::tempdir().unwrap();
7599        let output = format!("HEAD-SIGNAL\n{}TAIL-SIGNAL\n", "middle\n".repeat(2_000));
7600        let (_task_id, task) =
7601            insert_terminal_piped_task(&registry, &dir, "cat big.log", &output, "", false);
7602
7603        registry.post_terminal_transition(&task, true).unwrap();
7604        let completions = registry.drain_completions_for_session(Some("session"));
7605        assert_eq!(completions.len(), 1);
7606        let preview = &completions[0].output_preview;
7607        assert!(preview.contains("TAIL-SIGNAL"), "preview was {preview:?}");
7608        assert!(!preview.contains("HEAD-SIGNAL"), "preview was {preview:?}");
7609        assert!(completions[0].output_truncated);
7610    }
7611
7612    #[test]
7613    fn completion_preview_failure_keeps_head_and_tail() {
7614        // A FAILED task keeps a small head (first error / command banner) plus
7615        // a larger tail (tracebacks and summaries land at the end).
7616        let registry = BgTaskRegistry::default();
7617        let dir = tempfile::tempdir().unwrap();
7618        let output = format!("HEAD-SIGNAL\n{}TAIL-SIGNAL\n", "middle\n".repeat(2_000));
7619        let task_id = random_slug();
7620        let paths = task_paths(dir.path(), "session", &task_id).unwrap();
7621        fs::create_dir_all(&paths.dir).unwrap();
7622        fs::write(&paths.stdout, &output).unwrap();
7623        fs::write(&paths.stderr, "").unwrap();
7624        let mut metadata = PersistedTask::starting(
7625            task_id.clone(),
7626            "session".to_string(),
7627            "cat big.log".to_string(),
7628            dir.path().to_path_buf(),
7629            Some(dir.path().to_path_buf()),
7630            Some(30_000),
7631            true,
7632            false,
7633        );
7634        metadata.mark_terminal(BgTaskStatus::Failed, Some(1), None);
7635        write_task(&paths.json, &metadata).unwrap();
7636        registry
7637            .insert_rehydrated_task(metadata, paths, true)
7638            .expect("insert terminal task");
7639        let task = registry.task_for_session(&task_id, "session").unwrap();
7640
7641        registry.post_terminal_transition(&task, true).unwrap();
7642        let completions = registry.drain_completions_for_session(Some("session"));
7643        assert_eq!(completions.len(), 1);
7644        let preview = &completions[0].output_preview;
7645        assert!(preview.contains("HEAD-SIGNAL"), "preview was {preview:?}");
7646        assert!(preview.contains("TAIL-SIGNAL"), "preview was {preview:?}");
7647    }
7648
7649    #[test]
7650    fn has_completions_for_session_matches_pending_delivery() {
7651        let registry = BgTaskRegistry::default();
7652        assert!(!registry.has_completions_for_session(Some("session")));
7653        assert!(!registry.has_completions_for_session(None));
7654
7655        let dir = tempfile::tempdir().unwrap();
7656        let (_task_id, task) =
7657            insert_terminal_piped_task(&registry, &dir, QUICK_SUCCESS_COMMAND, "done\n", "", false);
7658        registry.post_terminal_transition(&task, true).unwrap();
7659
7660        assert!(registry.has_completions_for_session(Some("session")));
7661        assert!(registry.has_completions_for_session(None));
7662        assert!(!registry.has_completions_for_session(Some("other-session")));
7663
7664        let completions = registry.drain_completions_for_session(Some("session"));
7665        assert_eq!(completions.len(), 1);
7666        assert_eq!(completions[0].task_id, task.task_id);
7667    }
7668
7669    #[test]
7670    fn completion_drain_redelivers_until_ack_marks_it_delivered() {
7671        let registry = BgTaskRegistry::default();
7672        let dir = tempfile::tempdir().unwrap();
7673        let (task_id, task) =
7674            insert_terminal_piped_task(&registry, &dir, QUICK_SUCCESS_COMMAND, "done\n", "", false);
7675        registry.post_terminal_transition(&task, true).unwrap();
7676
7677        let first = registry.drain_completions_for_session(Some("session"));
7678        let second = registry.drain_completions_for_session(Some("session"));
7679        assert_eq!(first.len(), 1);
7680        assert_eq!(second.len(), 1);
7681        assert_eq!(first[0].task_id, task_id);
7682        assert_eq!(second[0].task_id, task_id);
7683
7684        let resolved =
7685            resolve_task_layout(&session_tasks_dir(dir.path(), "session"), &task_id).unwrap();
7686        assert!(!read_task_at(&resolved).unwrap().completion_delivered);
7687
7688        assert_eq!(
7689            registry.ack_completions_for_session(Some("session"), std::slice::from_ref(&task_id)),
7690            vec![task_id.clone()]
7691        );
7692        assert!(registry
7693            .drain_completions_for_session(Some("session"))
7694            .is_empty());
7695        assert!(read_task_at(&resolved).unwrap().completion_delivered);
7696    }
7697
7698    #[test]
7699    fn structured_gh_json_survives_intact_and_ignores_stderr() {
7700        let registry = BgTaskRegistry::default();
7701        let dir = tempfile::tempdir().unwrap();
7702        let calls = Arc::new(AtomicUsize::new(0));
7703        let calls_for_closure = Arc::clone(&calls);
7704        registry.set_compressor_with_exit_code(move |_command, output, _exit_code| {
7705            calls_for_closure.fetch_add(1, Ordering::SeqCst);
7706            CompressionResult::new(output)
7707        });
7708        let (task_id, _task) = insert_terminal_piped_task(
7709            &registry,
7710            &dir,
7711            "gh pr view 123 --json body",
7712            "{\"body\":\"hello\"}",
7713            "warning: stderr must not join json",
7714            true,
7715        );
7716
7717        let snapshot = registry
7718            .status(
7719                &task_id,
7720                "session",
7721                None,
7722                Some(dir.path()),
7723                RUNNING_OUTPUT_PREVIEW_BYTES,
7724            )
7725            .unwrap();
7726
7727        assert_eq!(snapshot.output_preview, "{\"body\":\"hello\"}");
7728        assert!(!snapshot.output_preview.contains("warning"));
7729        assert!(!snapshot.output_truncated);
7730        assert_eq!(
7731            calls.load(Ordering::SeqCst),
7732            0,
7733            "structured JSON bypasses compression"
7734        );
7735    }
7736
7737    #[test]
7738    fn registry_emits_single_recovery_marker_for_class_drops() {
7739        let registry = BgTaskRegistry::default();
7740        let dir = tempfile::tempdir().unwrap();
7741        registry.set_compressor_with_exit_code(move |_command, _output, _exit_code| {
7742            let mut dropped = BTreeMap::new();
7743            dropped.insert(DropClass::Error, 18);
7744            dropped.insert(DropClass::Warning, 6);
7745            CompressionResult::with_class_drops("kept diagnostic", dropped)
7746        });
7747        let (task_id, task) =
7748            insert_terminal_piped_task(&registry, &dir, "custom-tool", "raw", "", true);
7749
7750        let snapshot = registry
7751            .status(
7752                &task_id,
7753                "session",
7754                None,
7755                Some(dir.path()),
7756                RUNNING_OUTPUT_PREVIEW_BYTES,
7757            )
7758            .unwrap();
7759
7760        assert_eq!(snapshot.output_preview.matches("full output:").count(), 1);
7761        assert!(snapshot.output_preview.contains("+18 more errors"));
7762        assert!(snapshot.output_preview.contains("+6 more warnings"));
7763        assert!(snapshot
7764            .output_preview
7765            .contains(&format!("read \"{}\"", task.paths.stdout.display())));
7766        assert!(!snapshot.output_preview.contains("tail -n +"));
7767        assert!(snapshot.output_truncated);
7768    }
7769
7770    #[test]
7771    fn registry_marker_reports_semantic_and_byte_drops_once() {
7772        let registry = BgTaskRegistry::default();
7773        let dir = tempfile::tempdir().unwrap();
7774        registry.set_compressor_with_exit_code(move |_command, _output, _exit_code| {
7775            let mut dropped = BTreeMap::new();
7776            dropped.insert(DropClass::Error, 1);
7777            CompressionResult::with_class_drops(
7778                format!("HEAD-SIGNAL\n{}TAIL-SIGNAL", "middle\n".repeat(8_000)),
7779                dropped,
7780            )
7781        });
7782        let (task_id, _task) =
7783            insert_terminal_piped_task(&registry, &dir, "custom-tool", "raw", "", true);
7784
7785        let snapshot = registry
7786            .status(
7787                &task_id,
7788                "session",
7789                None,
7790                Some(dir.path()),
7791                RUNNING_OUTPUT_PREVIEW_BYTES,
7792            )
7793            .unwrap();
7794
7795        assert_eq!(snapshot.output_preview.matches("full output:").count(), 1);
7796        assert!(snapshot.output_preview.contains("+1 more error"));
7797        assert!(snapshot.output_preview.contains("truncated output"));
7798        assert!(snapshot.output_preview.contains("HEAD-SIGNAL"));
7799        assert!(snapshot.output_preview.contains("TAIL-SIGNAL"));
7800        assert!(!snapshot.output_preview.contains("...<truncated"));
7801        assert!(snapshot.output_truncated);
7802    }
7803
7804    #[test]
7805    fn cargo_stderr_class_drops_name_both_capture_paths() {
7806        let registry = BgTaskRegistry::default();
7807        let dir = tempfile::tempdir().unwrap();
7808        let filter_registry = crate::compress::toml_filter::FilterRegistry::default();
7809        registry.set_compressor_with_exit_code(move |command, output, exit_code| {
7810            crate::compress::compress_with_registry_exit_code(
7811                command,
7812                &output,
7813                exit_code,
7814                &filter_registry,
7815            )
7816        });
7817        let stderr = (0..22)
7818            .map(|index| {
7819                format!(
7820                    "error: cargo failure {index}\n  --> src/lib.rs:{}:1\n   |\n{} | boom\n",
7821                    index + 1,
7822                    index + 1
7823                )
7824            })
7825            .collect::<Vec<_>>()
7826            .join("\n");
7827        let (task_id, task) = insert_terminal_piped_task(
7828            &registry,
7829            &dir,
7830            "cargo check",
7831            "Finished dev [unoptimized] target(s) in 0.01s\n",
7832            &stderr,
7833            true,
7834        );
7835
7836        let snapshot = registry
7837            .status(
7838                &task_id,
7839                "session",
7840                None,
7841                Some(dir.path()),
7842                RUNNING_OUTPUT_PREVIEW_BYTES,
7843            )
7844            .unwrap();
7845
7846        assert!(snapshot.output_preview.contains("+2 more errors"));
7847        assert!(snapshot
7848            .output_preview
7849            .contains(&format!("read \"{}\"", task.paths.stdout.display())));
7850        assert!(snapshot
7851            .output_preview
7852            .contains(&format!("read \"{}\"", task.paths.stderr.display())));
7853        assert!(!snapshot.output_preview.contains("tail -n +"));
7854    }
7855
7856    #[test]
7857    fn over_ceiling_structured_json_uses_pointer_not_partial_json() {
7858        let registry = BgTaskRegistry::default();
7859        let dir = tempfile::tempdir().unwrap();
7860        let body = format!("{{\"body\":\"{}\"}}", "x".repeat(60 * 1024));
7861        let (task_id, task) = insert_terminal_piped_task(
7862            &registry,
7863            &dir,
7864            "cd /repo && gh pr view 123 --json body",
7865            &body,
7866            "",
7867            true,
7868        );
7869
7870        let snapshot = registry
7871            .status(
7872                &task_id,
7873                "session",
7874                None,
7875                Some(dir.path()),
7876                RUNNING_OUTPUT_PREVIEW_BYTES,
7877            )
7878            .unwrap();
7879
7880        assert!(snapshot.output_preview.starts_with("[JSON output "));
7881        assert!(snapshot
7882            .output_preview
7883            .contains(&task.paths.stdout.display().to_string()));
7884        assert!(!snapshot.output_preview.contains(&"x".repeat(1024)));
7885        assert!(snapshot.output_truncated);
7886    }
7887
7888    #[test]
7889    fn toml_strip_tail_cap_uses_full_output_hint_not_offset_hint() {
7890        let registry = BgTaskRegistry::default();
7891        let dir = tempfile::tempdir().unwrap();
7892        let filter_registry = crate::compress::toml_filter::build_registry(
7893            crate::compress::builtin_filters::ALL,
7894            None,
7895            None,
7896        );
7897        registry.set_compressor_with_exit_code(move |command, output, exit_code| {
7898            crate::compress::compress_with_registry_exit_code(
7899                command,
7900                &output,
7901                exit_code,
7902                &filter_registry,
7903            )
7904        });
7905        let stdout = format!(
7906            "make[1]: Entering directory `/tmp`\n{}",
7907            (0..100)
7908                .map(|index| format!("compile line {index}"))
7909                .collect::<Vec<_>>()
7910                .join("\n")
7911        );
7912        let (task_id, task) =
7913            insert_terminal_piped_task(&registry, &dir, "make all", &stdout, "", true);
7914
7915        let snapshot = registry
7916            .status(
7917                &task_id,
7918                "session",
7919                None,
7920                Some(dir.path()),
7921                RUNNING_OUTPUT_PREVIEW_BYTES,
7922            )
7923            .unwrap();
7924
7925        assert!(snapshot.output_preview.contains("compile line 99"));
7926        assert!(snapshot.output_preview.contains(&format!(
7927            "full output: read \"{}\"",
7928            task.paths.stdout.display()
7929        )));
7930        assert!(!snapshot
7931            .output_preview
7932            .contains(&format!("read \"{}\"", task.paths.stderr.display())));
7933        assert!(!snapshot.output_preview.contains("tail -n +"));
7934    }
7935
7936    #[test]
7937    fn compressed_false_raw_passthrough_uses_wider_head_tail_cap() {
7938        let registry = BgTaskRegistry::default();
7939        let dir = tempfile::tempdir().unwrap();
7940        let output = format!("RAW-HEAD\n{}RAW-TAIL\n", "raw-middle\n".repeat(8_000));
7941        let (task_id, task) =
7942            insert_terminal_piped_task(&registry, &dir, "cat raw.log", &output, "RAW-ERR\n", false);
7943
7944        let snapshot = registry
7945            .status(
7946                &task_id,
7947                "session",
7948                None,
7949                Some(dir.path()),
7950                RUNNING_OUTPUT_PREVIEW_BYTES,
7951            )
7952            .unwrap();
7953
7954        assert!(snapshot.output_preview.contains("RAW-HEAD"));
7955        assert!(snapshot.output_preview.contains("RAW-TAIL"));
7956        assert!(snapshot.output_preview.contains("truncated output"));
7957        assert!(snapshot
7958            .output_preview
7959            .contains(&format!("read \"{}\"", task.paths.stdout.display())));
7960        assert!(snapshot
7961            .output_preview
7962            .contains(&format!("read \"{}\"", task.paths.stderr.display())));
7963        assert!(!snapshot.output_preview.contains("tail -n +"));
7964        assert!(snapshot.output_preview.len() > 16 * 1024);
7965        assert!(snapshot.output_truncated);
7966    }
7967
7968    #[test]
7969    fn pty_terminal_snapshot_bypasses_line_compression() {
7970        let registry = BgTaskRegistry::default();
7971        let dir = tempfile::tempdir().unwrap();
7972        let calls = Arc::new(AtomicUsize::new(0));
7973        let calls_for_closure = Arc::clone(&calls);
7974        registry.set_compressor_with_exit_code(move |_command, output, _exit_code| {
7975            calls_for_closure.fetch_add(1, Ordering::SeqCst);
7976            CompressionResult::new(output)
7977        });
7978        let (task_id, _task) = insert_terminal_pty_task(&registry, &dir, "raw\u{1b}[31m pty bytes");
7979
7980        let snapshot = registry
7981            .status(
7982                &task_id,
7983                "session",
7984                None,
7985                Some(dir.path()),
7986                RUNNING_OUTPUT_PREVIEW_BYTES,
7987            )
7988            .unwrap();
7989
7990        assert_eq!(snapshot.info.mode, BgMode::Pty);
7991        assert_eq!(snapshot.output_preview, "");
7992        assert_eq!(calls.load(Ordering::SeqCst), 0);
7993    }
7994
7995    #[test]
7996    fn pty_dimensions_are_persisted_and_returned_in_snapshot() {
7997        let registry = BgTaskRegistry::default();
7998        let dir = tempfile::tempdir().unwrap();
7999        let task_id = registry
8000            .spawn_pty(
8001                SpawnPlan::Unsandboxed,
8002                QUICK_SUCCESS_COMMAND,
8003                "session".to_string(),
8004                dir.path().to_path_buf(),
8005                HashMap::new(),
8006                Some(Duration::from_secs(30)),
8007                dir.path().to_path_buf(),
8008                10,
8009                true,
8010                false,
8011                Some(dir.path().to_path_buf()),
8012                50,
8013                120,
8014            )
8015            .unwrap();
8016
8017        let resolved =
8018            resolve_task_layout(&session_tasks_dir(dir.path(), "session"), &task_id).unwrap();
8019        let metadata = read_task_at(&resolved).unwrap();
8020        assert_eq!(
8021            metadata.schema_version,
8022            crate::bash_background::persistence::SCHEMA_VERSION
8023        );
8024        assert_eq!(metadata.mode, BgMode::Pty);
8025        assert_eq!(metadata.pty_rows, Some(50));
8026        assert_eq!(metadata.pty_cols, Some(120));
8027
8028        let snapshot = registry
8029            .status(&task_id, "session", None, Some(dir.path()), 1024)
8030            .unwrap();
8031        assert_eq!(snapshot.pty_rows, Some(50));
8032        assert_eq!(snapshot.pty_cols, Some(120));
8033    }
8034
8035    /// Spawn a child process that exits immediately and return it after
8036    /// it has terminated. Used by reap_child tests to simulate the
8037    /// "child exists and is dead" state when the watchdog has already
8038    /// nulled out the original child handle.
8039    fn spawn_dead_child() -> std::process::Child {
8040        #[cfg(unix)]
8041        let mut cmd = std::process::Command::new("true");
8042        #[cfg(windows)]
8043        let mut cmd = {
8044            let mut c = std::process::Command::new("cmd");
8045            c.args(["/c", "exit", "0"]);
8046            c
8047        };
8048        cmd.stdin(std::process::Stdio::null());
8049        cmd.stdout(std::process::Stdio::null());
8050        cmd.stderr(std::process::Stdio::null());
8051        let mut child = cmd.spawn().expect("spawn replacement child for reap test");
8052        // Poll try_wait() until the child actually exits, instead of calling
8053        // wait() which closes the OS handle. On Windows, after wait()
8054        // closes the handle, subsequent try_wait() calls (which reap_child
8055        // depends on) return Err — the test was inadvertently giving
8056        // reap_child an unusable child handle. Polling try_wait() keeps the
8057        // handle open and observes natural exit, matching the production
8058        // shape where the watchdog discovers an exited child for the first
8059        // time.
8060        let started = Instant::now();
8061        loop {
8062            match child.try_wait() {
8063                Ok(Some(_)) => break,
8064                Ok(None) => {
8065                    if started.elapsed() > CHILD_EXIT_LIVENESS_BOUND {
8066                        panic!("dead-child stand-in did not exit within the liveness bound");
8067                    }
8068                    std::thread::sleep(Duration::from_millis(10));
8069                }
8070                Err(error) => panic!("dead-child try_wait failed: {error}"),
8071            }
8072        }
8073        child
8074    }
8075
8076    #[test]
8077    fn ack_marks_delivered_even_when_completion_was_already_consumed_locally() {
8078        let registry = BgTaskRegistry::default();
8079        let dir = tempfile::tempdir().unwrap();
8080        let task_id = registry
8081            .spawn(
8082                SpawnPlan::Unsandboxed,
8083                LONG_RUNNING_COMMAND,
8084                "session".to_string(),
8085                dir.path().to_path_buf(),
8086                HashMap::new(),
8087                Some(Duration::from_secs(30)),
8088                dir.path().to_path_buf(),
8089                10,
8090                true,
8091                false,
8092                Some(dir.path().to_path_buf()),
8093            )
8094            .unwrap();
8095        registry
8096            .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
8097            .unwrap();
8098        assert_eq!(
8099            registry
8100                .drain_completions_for_session(Some("session"))
8101                .len(),
8102            1
8103        );
8104
8105        // Simulate the plugin consuming a sync bash_watch({ exit:true }) result
8106        // locally before the Rust completion queue is drained/acked.
8107        registry.inner.completions.lock().unwrap().clear();
8108
8109        assert_eq!(
8110            registry.ack_completions_for_session(Some("session"), std::slice::from_ref(&task_id)),
8111            vec![task_id.clone()]
8112        );
8113        assert!(registry
8114            .drain_completions_for_session(Some("session"))
8115            .is_empty());
8116
8117        let resolved =
8118            resolve_task_layout(&session_tasks_dir(dir.path(), "session"), &task_id).unwrap();
8119        let metadata = read_task_at(&resolved).unwrap();
8120        assert!(metadata.completion_delivered);
8121
8122        let replayed = BgTaskRegistry::default();
8123        replayed
8124            .replay_session_inner(dir.path(), "session", None)
8125            .unwrap();
8126        assert!(replayed
8127            .drain_completions_for_session(Some("session"))
8128            .is_empty());
8129    }
8130
8131    #[test]
8132    fn reclaimed_root_kills_running_task_and_persists_reason() {
8133        let registry = BgTaskRegistry::default();
8134        let root = tempfile::tempdir().unwrap();
8135        let storage = tempfile::tempdir().unwrap();
8136        let task_id = registry
8137            .spawn(
8138                SpawnPlan::Unsandboxed,
8139                LONG_RUNNING_COMMAND,
8140                "session".to_string(),
8141                root.path().to_path_buf(),
8142                HashMap::new(),
8143                Some(Duration::from_secs(30)),
8144                storage.path().to_path_buf(),
8145                10,
8146                true,
8147                false,
8148                Some(root.path().to_path_buf()),
8149            )
8150            .unwrap();
8151        let pid = registry
8152            .status(
8153                &task_id,
8154                "session",
8155                Some(root.path()),
8156                Some(storage.path()),
8157                0,
8158            )
8159            .unwrap()
8160            .child_pid
8161            .unwrap();
8162        assert!(is_process_alive(pid));
8163
8164        assert_eq!(registry.kill_running_tasks_for_root(root.path()), 1);
8165        let deadline = Instant::now() + CHILD_EXIT_LIVENESS_BOUND;
8166        while is_process_alive(pid) {
8167            assert!(
8168                Instant::now() < deadline,
8169                "reclaimed task process survived kill"
8170            );
8171            std::thread::sleep(Duration::from_millis(20));
8172        }
8173
8174        let snapshot = registry
8175            .status(
8176                &task_id,
8177                "session",
8178                Some(root.path()),
8179                Some(storage.path()),
8180                0,
8181            )
8182            .unwrap();
8183        assert_eq!(snapshot.info.status, BgTaskStatus::Killed);
8184        assert_eq!(
8185            snapshot.info.status_reason.as_deref(),
8186            Some(ROOT_RECLAIMED_REASON)
8187        );
8188        let persisted = read_task(
8189            &registry
8190                .task_json_path(&task_id, "session")
8191                .expect("reclaimed task metadata path"),
8192        )
8193        .expect("persisted reclaimed task");
8194        assert_eq!(
8195            persisted.status_reason.as_deref(),
8196            Some(ROOT_RECLAIMED_REASON)
8197        );
8198        let completion = registry
8199            .drain_completions_for_session(Some("session"))
8200            .pop()
8201            .expect("reclaimed task completion");
8202        assert_eq!(
8203            completion.status_reason.as_deref(),
8204            Some(ROOT_RECLAIMED_REASON)
8205        );
8206        registry.detach();
8207    }
8208
8209    #[test]
8210    fn reclaimed_root_kills_pty_task_and_preserves_reason() {
8211        let registry = BgTaskRegistry::default();
8212        let root = tempfile::tempdir().unwrap();
8213        let storage = tempfile::tempdir().unwrap();
8214        let command = if cfg!(windows) {
8215            "Start-Sleep -Seconds 30"
8216        } else {
8217            "sleep 30"
8218        };
8219        let task_id = registry
8220            .spawn_pty(
8221                SpawnPlan::Unsandboxed,
8222                command,
8223                "session".to_string(),
8224                root.path().to_path_buf(),
8225                HashMap::new(),
8226                Some(Duration::from_secs(60)),
8227                storage.path().to_path_buf(),
8228                10,
8229                true,
8230                false,
8231                Some(root.path().to_path_buf()),
8232                24,
8233                80,
8234            )
8235            .unwrap();
8236        let pid = registry
8237            .status(
8238                &task_id,
8239                "session",
8240                Some(root.path()),
8241                Some(storage.path()),
8242                0,
8243            )
8244            .unwrap()
8245            .child_pid
8246            .unwrap();
8247        assert!(is_process_alive(pid));
8248
8249        assert_eq!(registry.kill_running_tasks_for_root(root.path()), 1);
8250        let deadline = Instant::now() + Duration::from_secs(10);
8251        loop {
8252            let snapshot = registry
8253                .status(
8254                    &task_id,
8255                    "session",
8256                    Some(root.path()),
8257                    Some(storage.path()),
8258                    0,
8259                )
8260                .unwrap();
8261            if snapshot.info.status.is_terminal() {
8262                assert_eq!(snapshot.info.status, BgTaskStatus::Killed);
8263                assert_eq!(
8264                    snapshot.info.status_reason.as_deref(),
8265                    Some(ROOT_RECLAIMED_REASON)
8266                );
8267                break;
8268            }
8269            assert!(
8270                Instant::now() < deadline,
8271                "reclaimed PTY task did not terminate"
8272            );
8273            std::thread::sleep(Duration::from_millis(20));
8274        }
8275        assert!(!is_process_alive(pid));
8276        // The terminal status and the completion frame are written by
8277        // different actors (kill finalize vs the watchdog's terminal-transition
8278        // scan), so the completion can trail the observed Killed status. Poll
8279        // the drain rather than reading it once.
8280        let completion = loop {
8281            if let Some(completion) = registry
8282                .drain_completions_for_session(Some("session"))
8283                .pop()
8284            {
8285                break completion;
8286            }
8287            assert!(
8288                Instant::now() < deadline,
8289                "reclaimed PTY completion never arrived"
8290            );
8291            std::thread::sleep(Duration::from_millis(20));
8292        };
8293        assert_eq!(
8294            completion.status_reason.as_deref(),
8295            Some(ROOT_RECLAIMED_REASON)
8296        );
8297        registry.detach();
8298    }
8299
8300    #[test]
8301    fn register_watch_rejects_unknown_task() {
8302        let registry = BgTaskRegistry::default();
8303
8304        let result = registry.register_watch(
8305            "missing-task".to_string(),
8306            WatchPattern::Substring("READY".into()),
8307            true,
8308        );
8309
8310        assert_eq!(result, Err("task_not_found"));
8311    }
8312
8313    #[test]
8314    fn register_watch_on_terminal_task_scans_existing_output() {
8315        let frames = Arc::new(Mutex::new(Vec::new()));
8316        let captured = Arc::clone(&frames);
8317        let sender: crate::context::ProgressSender = Arc::new(Box::new(move |frame| {
8318            captured.lock().unwrap().push(frame);
8319        })
8320            as Box<dyn Fn(PushFrame) + Send + Sync>);
8321        let registry = BgTaskRegistry::new(Arc::new(Mutex::new(Some(sender))));
8322        let dir = tempfile::tempdir().unwrap();
8323        let task_id = registry
8324            .spawn(
8325                SpawnPlan::Unsandboxed,
8326                LONG_RUNNING_COMMAND,
8327                "session".to_string(),
8328                dir.path().to_path_buf(),
8329                HashMap::new(),
8330                Some(Duration::from_secs(30)),
8331                dir.path().to_path_buf(),
8332                10,
8333                true,
8334                false,
8335                Some(dir.path().to_path_buf()),
8336            )
8337            .unwrap();
8338        registry
8339            .inner
8340            .shutdown
8341            .store(true, std::sync::atomic::Ordering::SeqCst);
8342        let task = registry.task_for_session(&task_id, "session").unwrap();
8343        std::fs::write(&task.paths.stdout, "READY\n").unwrap();
8344        registry
8345            .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
8346            .unwrap();
8347        frames.lock().unwrap().clear();
8348        registry.inner.completions.lock().unwrap().clear();
8349
8350        registry
8351            .register_watch(
8352                task_id.clone(),
8353                WatchPattern::Substring("READY".into()),
8354                true,
8355            )
8356            .unwrap();
8357
8358        let frames = frames.lock().unwrap();
8359        let frame = frames
8360            .iter()
8361            .find_map(|frame| match frame {
8362                PushFrame::BashPatternMatch(frame) => Some(frame),
8363                _ => None,
8364            })
8365            .expect("terminal watch registration should emit pattern frame");
8366        assert_eq!(frame.reason, "pattern_match");
8367        assert_eq!(frame.task_id, task_id);
8368        assert_eq!(frame.session_id, "session");
8369        assert_eq!(frame.match_text, "READY");
8370        assert_eq!(frame.match_offset, 0);
8371        assert_eq!(registry.active_watch_count(&frame.task_id), 0);
8372        let metadata = read_task(&task.paths.json).unwrap();
8373        assert!(
8374            !metadata.completion_delivered,
8375            "terminal pattern notification remains unacked until explicit ack"
8376        );
8377        drop(frames);
8378        assert_eq!(
8379            registry.ack_completions_for_session(Some("session"), std::slice::from_ref(&task_id),),
8380            vec![task_id]
8381        );
8382        assert!(read_task(&task.paths.json).unwrap().completion_delivered);
8383    }
8384
8385    #[test]
8386    fn cleanup_finished_removes_terminal_tasks_older_than_threshold() {
8387        let registry = BgTaskRegistry::default();
8388        let dir = tempfile::tempdir().unwrap();
8389        let task_id = registry
8390            .spawn(
8391                SpawnPlan::Unsandboxed,
8392                QUICK_SUCCESS_COMMAND,
8393                "session".to_string(),
8394                dir.path().to_path_buf(),
8395                HashMap::new(),
8396                Some(Duration::from_secs(30)),
8397                dir.path().to_path_buf(),
8398                10,
8399                true,
8400                false,
8401                Some(dir.path().to_path_buf()),
8402            )
8403            .unwrap();
8404        registry
8405            .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
8406            .unwrap();
8407        let completions = registry.drain_completions_for_session(Some("session"));
8408        assert_eq!(completions.len(), 1);
8409        assert_eq!(
8410            registry.ack_completions_for_session(Some("session"), std::slice::from_ref(&task_id)),
8411            vec![task_id.clone()]
8412        );
8413
8414        registry.cleanup_finished(Duration::ZERO);
8415
8416        assert!(registry.inner.tasks.lock().unwrap().is_empty());
8417    }
8418
8419    #[test]
8420    fn cleanup_finished_retains_undelivered_terminals() {
8421        let registry = BgTaskRegistry::default();
8422        let dir = tempfile::tempdir().unwrap();
8423        let task_id = registry
8424            .spawn(
8425                SpawnPlan::Unsandboxed,
8426                QUICK_SUCCESS_COMMAND,
8427                "session".to_string(),
8428                dir.path().to_path_buf(),
8429                HashMap::new(),
8430                Some(Duration::from_secs(30)),
8431                dir.path().to_path_buf(),
8432                10,
8433                true,
8434                false,
8435                Some(dir.path().to_path_buf()),
8436            )
8437            .unwrap();
8438        registry
8439            .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
8440            .unwrap();
8441
8442        registry.cleanup_finished(Duration::ZERO);
8443
8444        assert!(registry.inner.tasks.lock().unwrap().contains_key(&task_id));
8445    }
8446
8447    /// Verify that the live watchdog path (reap_child) gives an exited
8448    /// child one watchdog pass for its exit marker to land, then marks the
8449    /// task Failed if the next pass still sees no marker.
8450    ///
8451    /// Cross-platform: uses a quick-exiting command that does NOT go
8452    /// through the wrapper script (we manually clear the exit marker
8453    /// after spawn to simulate the wrapper crashing before write).
8454    #[test]
8455    fn reap_child_marks_failed_when_child_exits_without_exit_marker() {
8456        let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
8457        let dir = tempfile::tempdir().unwrap();
8458        let task_id = registry
8459            .spawn(
8460                SpawnPlan::Unsandboxed,
8461                QUICK_SUCCESS_COMMAND,
8462                "session".to_string(),
8463                dir.path().to_path_buf(),
8464                HashMap::new(),
8465                Some(Duration::from_secs(30)),
8466                dir.path().to_path_buf(),
8467                10,
8468                true,
8469                false,
8470                Some(dir.path().to_path_buf()),
8471            )
8472            .unwrap();
8473
8474        let task = registry.task_for_session(&task_id, "session").unwrap();
8475
8476        // Wait for the child to actually exit and the wrapper to either
8477        // write the marker or fail. Then nuke the marker to simulate
8478        // wrapper crash before write. Poll up to 5s; this is plenty for a
8479        // `true`/`cmd /c exit 0` invocation.
8480        let started = Instant::now();
8481        loop {
8482            let exited = {
8483                let mut state = task.state.lock().unwrap();
8484                match &mut state.runtime {
8485                    TaskRuntime::Piped(Some(child)) => matches!(child.try_wait(), Ok(Some(_))),
8486                    _ => true,
8487                }
8488            };
8489            if exited {
8490                break;
8491            }
8492            assert!(
8493                started.elapsed() < CHILD_EXIT_LIVENESS_BOUND,
8494                "child should exit quickly"
8495            );
8496            std::thread::sleep(Duration::from_millis(20));
8497        }
8498
8499        // Stop the watchdog so it doesn't race with our manual reap_child.
8500        // On fast Windows runners the watchdog ticks (every 500ms) can
8501        // observe the child exit and reap it before this test's assertion
8502        // fires, leaving us with state.child = None and an already-terminal
8503        // status. We specifically want to test reap_child's logic when
8504        // invoked manually on a Running-but-actually-dead task, so we need
8505        // exclusive control over the reap path here.
8506        registry
8507            .inner
8508            .shutdown
8509            .store(true, std::sync::atomic::Ordering::SeqCst);
8510        // Give the watchdog at most one tick (500ms) to notice shutdown
8511        // before we touch task state. Without this, an in-flight watchdog
8512        // iteration could still race with our state setup below.
8513        std::thread::sleep(Duration::from_millis(550));
8514
8515        // Wrapper likely wrote the marker by now; remove it to simulate
8516        // a wrapper crash that exited before persisting the exit code.
8517        let _ = std::fs::remove_file(&task.paths.exit);
8518
8519        // The watchdog may have already reaped the child handle and
8520        // marked the task terminal before we got here. Reset both so
8521        // reap_child has the "Running task whose child just exited"
8522        // shape it's designed to handle. If the original child handle is
8523        // gone, install a quick-exited stand-in so the first reap exercises
8524        // the same try_wait path as production.
8525        //
8526        // CRITICAL on Windows: the watchdog ticks fast enough that the
8527        // JSON on disk may already say `Completed`. `update_task` (called
8528        // by `reap_child`) reads from disk, applies the closure, but
8529        // ROLLS BACK if the original on-disk state was already terminal
8530        // (see persistence.rs::update_task). So we must reset BOTH
8531        // in-memory metadata AND the JSON on disk to a Running state to
8532        // give reap_child the fresh shape it expects to operate on.
8533        {
8534            let mut state = task.state.lock().unwrap();
8535            state.metadata.status = BgTaskStatus::Running;
8536            state.metadata.status_reason = None;
8537            state.metadata.exit_code = None;
8538            state.metadata.finished_at = None;
8539            state.metadata.duration_ms = None;
8540            // Persist the reset state to disk so update_task's terminal
8541            // rollback guard sees a non-terminal starting point.
8542            crate::bash_background::persistence::write_task(&task.paths.json, &state.metadata)
8543                .expect("persist reset Running metadata for reap_child test");
8544            // If the watchdog already nulled state.child, we need to
8545            // simulate "child exists and is dead" so reap_child's
8546            // try_wait path runs. Spawn a quick-exit child as a stand-in.
8547            if matches!(state.runtime, TaskRuntime::Piped(None)) {
8548                state.runtime = TaskRuntime::Piped(Some(spawn_dead_child()));
8549            }
8550        }
8551        // Clear the terminal_at marker too so mark_terminal_now() can fire
8552        // again inside reap_child.
8553        *task.terminal_at.lock().unwrap() = None;
8554
8555        // Sanity: task is still Running per metadata (replay/poll hasn't
8556        // observed the missing marker yet).
8557        assert!(
8558            task.is_running(),
8559            "precondition: metadata.status == Running"
8560        );
8561        assert!(
8562            !task.paths.exit.exists(),
8563            "precondition: exit marker absent"
8564        );
8565
8566        // First watchdog observation is intentionally insufficient to
8567        // declare failure. A missing marker may just mean the wrapper is
8568        // still completing its tmp-file-to-marker rename, so reap_child only
8569        // drops the child handle and switches to detached PID monitoring.
8570        registry.reap_child(&task);
8571
8572        {
8573            let state = task.state.lock().unwrap();
8574            assert_eq!(
8575                state.metadata.status,
8576                BgTaskStatus::Running,
8577                "first reap must leave status Running while waiting one pass for marker"
8578            );
8579            assert_eq!(
8580                state.metadata.status_reason, None,
8581                "first reap must not record a failure reason"
8582            );
8583            assert!(
8584                matches!(state.runtime, TaskRuntime::Piped(None)),
8585                "child handle must be released after first reap"
8586            );
8587            assert!(
8588                state.detached,
8589                "task must be marked detached after first reap"
8590            );
8591        }
8592
8593        // Second watchdog observation sees the detached PID is dead and the
8594        // marker is still absent. That is strong enough evidence that the
8595        // wrapper exited without persisting an exit code.
8596        registry.reap_child(&task);
8597
8598        let state = task.state.lock().unwrap();
8599        assert!(
8600            state.metadata.status.is_terminal(),
8601            "second reap must transition to terminal when PID dead and no marker. Got status={:?}",
8602            state.metadata.status
8603        );
8604        assert_eq!(
8605            state.metadata.status,
8606            BgTaskStatus::Failed,
8607            "must specifically be Failed (not Killed): status={:?}",
8608            state.metadata.status
8609        );
8610        assert_eq!(
8611            state.metadata.status_reason.as_deref(),
8612            Some("process exited without exit marker"),
8613            "reason must match replay path's wording: {:?}",
8614            state.metadata.status_reason
8615        );
8616        assert!(
8617            matches!(state.runtime, TaskRuntime::Piped(None)),
8618            "child handle must stay released after second reap"
8619        );
8620        assert!(
8621            state.detached,
8622            "task must remain detached after second reap"
8623        );
8624    }
8625
8626    /// Companion to the above: when the exit marker DOES exist on disk
8627    /// at reap_child time, reap_child must NOT mark the task Failed.
8628    /// Instead it leaves status=Running and lets the next poll_task()
8629    /// cycle finalize via the marker.
8630    #[test]
8631    fn reap_child_preserves_running_when_exit_marker_exists() {
8632        let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
8633        let dir = tempfile::tempdir().unwrap();
8634        let task_id = registry
8635            .spawn(
8636                SpawnPlan::Unsandboxed,
8637                QUICK_SUCCESS_COMMAND,
8638                "session".to_string(),
8639                dir.path().to_path_buf(),
8640                HashMap::new(),
8641                Some(Duration::from_secs(30)),
8642                dir.path().to_path_buf(),
8643                10,
8644                true,
8645                false,
8646                Some(dir.path().to_path_buf()),
8647            )
8648            .unwrap();
8649
8650        let task = registry.task_for_session(&task_id, "session").unwrap();
8651
8652        // Wait for child to exit AND for the marker to land. Both happen
8653        // shortly after the wrapper finishes — but we want both observed.
8654        let started = Instant::now();
8655        loop {
8656            let exited = {
8657                let mut state = task.state.lock().unwrap();
8658                match &mut state.runtime {
8659                    TaskRuntime::Piped(Some(child)) => matches!(child.try_wait(), Ok(Some(_))),
8660                    _ => true,
8661                }
8662            };
8663            if exited && task.paths.exit.exists() {
8664                break;
8665            }
8666            assert!(
8667                started.elapsed() < CHILD_EXIT_LIVENESS_BOUND,
8668                "child should exit and write marker quickly"
8669            );
8670            std::thread::sleep(Duration::from_millis(20));
8671        }
8672
8673        // Stop the watchdog so it doesn't race with our manual reap_child.
8674        // On fast Windows runners the watchdog can call poll_task (which
8675        // finalizes via marker) before this test asserts the
8676        // "marker exists, status still Running" invariant. We want
8677        // exclusive control over the reap path.
8678        registry
8679            .inner
8680            .shutdown
8681            .store(true, std::sync::atomic::Ordering::SeqCst);
8682        std::thread::sleep(Duration::from_millis(550));
8683
8684        // If the watchdog already finalized the task before we stopped it,
8685        // restore the test setup: reset status to Running and ensure the
8686        // marker file is still on disk. We're testing reap_child's
8687        // behavior when called manually with both child-exited AND
8688        // marker-present, regardless of whether the watchdog beat us.
8689        {
8690            let mut state = task.state.lock().unwrap();
8691            state.metadata.status = BgTaskStatus::Running;
8692            state.metadata.status_reason = None;
8693            if matches!(state.runtime, TaskRuntime::Piped(None)) {
8694                state.runtime = TaskRuntime::Piped(Some(spawn_dead_child()));
8695            }
8696        }
8697        *task.terminal_at.lock().unwrap() = None;
8698        // Make sure the marker is still on disk (poll_task removes it on
8699        // finalization). Recreate it if needed.
8700        if !task.paths.exit.exists() {
8701            std::fs::write(&task.paths.exit, "0").expect("write replacement exit marker");
8702        }
8703
8704        // reap_child sees: child exited, marker exists. It should:
8705        //  - drop state.child / set state.detached = true
8706        //  - NOT change status (poll_task will finalize via marker next tick)
8707        registry.reap_child(&task);
8708
8709        let state = task.state.lock().unwrap();
8710        assert!(
8711            matches!(state.runtime, TaskRuntime::Piped(None)),
8712            "child handle still released even when marker exists"
8713        );
8714        assert!(
8715            state.detached,
8716            "task still marked detached even when marker exists"
8717        );
8718        // Status remains Running because reap_child defers to poll_task
8719        // when a marker exists. It would be wrong for reap to record the
8720        // marker outcome (poll_task does that with proper exit-code
8721        // parsing).
8722        assert_eq!(
8723            state.metadata.status,
8724            BgTaskStatus::Running,
8725            "reap_child must defer to poll_task when marker exists"
8726        );
8727    }
8728
8729    /// Read a process's `ps` state string ("Z", "S", "R", etc). Returns
8730    /// `None` once the PID has been fully reaped (no row), which is the
8731    /// post-reap state we want.
8732    #[cfg(unix)]
8733    fn pid_stat(pid: u32) -> Option<String> {
8734        let output = std::process::Command::new("ps")
8735            .args(["-o", "stat=", "-p", &pid.to_string()])
8736            .output()
8737            .ok()?;
8738        if !output.status.success() {
8739            return None;
8740        }
8741        let stat = String::from_utf8_lossy(&output.stdout).trim().to_string();
8742        if stat.is_empty() {
8743            None
8744        } else {
8745            Some(stat)
8746        }
8747    }
8748
8749    /// A `<defunct>` zombie carries `ps` state starting with 'Z'.
8750    #[cfg(unix)]
8751    fn is_zombie(pid: u32) -> bool {
8752        pid_stat(pid).is_some_and(|stat| stat.starts_with('Z'))
8753    }
8754
8755    /// Spawn a child that exits immediately and wait — via `ps`, NOT
8756    /// `try_wait()`/`wait()` — until it is observably a `<defunct>` zombie,
8757    /// then return the still-unreaped handle. This reproduces the exact
8758    /// state issue #91 leaves behind: an exited OS child whose parent has
8759    /// not reaped it.
8760    #[cfg(unix)]
8761    fn spawn_unreaped_zombie() -> std::process::Child {
8762        let child = std::process::Command::new("true")
8763            .stdin(std::process::Stdio::null())
8764            .stdout(std::process::Stdio::null())
8765            .stderr(std::process::Stdio::null())
8766            .spawn()
8767            .expect("spawn zombie stand-in");
8768        let pid = child.id();
8769        let started = Instant::now();
8770        while !is_zombie(pid) {
8771            assert!(
8772                started.elapsed() < CHILD_EXIT_LIVENESS_BOUND,
8773                "stand-in child should become a zombie within the liveness bound"
8774            );
8775            std::thread::sleep(Duration::from_millis(10));
8776        }
8777        // Return WITHOUT reaping — the handle still owns an unwaited zombie.
8778        child
8779    }
8780
8781    /// Regression test for issue #91: the exit-marker terminal path
8782    /// (`poll_task` -> `finalize_from_marker`) must REAP the direct child
8783    /// handle, not merely drop it. Dropping a `std::process::Child` does not
8784    /// `wait()` on Unix, so the exited child lingers as a `[mv] <defunct>`
8785    /// zombie until AFT exits.
8786    ///
8787    /// We install a known-unreaped zombie into the task's child slot and
8788    /// drive the marker finalize path, then assert the child is gone (reaped)
8789    /// rather than still `<defunct>`.
8790    #[cfg(unix)]
8791    #[test]
8792    fn finalize_from_marker_reaps_child_no_zombie() {
8793        use std::sync::atomic::Ordering;
8794
8795        let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
8796        let dir = tempfile::tempdir().unwrap();
8797        let task_id = registry
8798            .spawn(
8799                SpawnPlan::Unsandboxed,
8800                QUICK_SUCCESS_COMMAND,
8801                "session".to_string(),
8802                dir.path().to_path_buf(),
8803                HashMap::new(),
8804                Some(Duration::from_secs(30)),
8805                dir.path().to_path_buf(),
8806                10,
8807                true,
8808                false,
8809                Some(dir.path().to_path_buf()),
8810            )
8811            .unwrap();
8812
8813        // Stop the watchdog so the ONLY terminal-transition path under test
8814        // is the exit-marker finalize (not reap_child's try_wait, which would
8815        // reap the child for us and mask the bug).
8816        registry.inner.shutdown.store(true, Ordering::SeqCst);
8817        std::thread::sleep(Duration::from_millis(550));
8818
8819        let task = registry.task_for_session(&task_id, "session").unwrap();
8820
8821        // Wait for the wrapper's exit marker to land. We deliberately do NOT
8822        // call try_wait()/wait() on the real child here — doing so would reap
8823        // it and defeat the test.
8824        let started = Instant::now();
8825        while !task.paths.exit.exists() {
8826            assert!(
8827                started.elapsed() < CHILD_EXIT_LIVENESS_BOUND,
8828                "exit marker should land quickly for `true`"
8829            );
8830            std::thread::sleep(Duration::from_millis(20));
8831        }
8832
8833        // Reset to a fresh Running shape and install a guaranteed-unreaped
8834        // zombie as the child handle, so the finalize path's reap behavior is
8835        // exercised deterministically regardless of how the real child was
8836        // handled. Persist Running so update_task's terminal-rollback guard
8837        // sees a non-terminal starting point.
8838        let zombie_pid;
8839        {
8840            let mut state = task.state.lock().unwrap();
8841            state.metadata.status = BgTaskStatus::Running;
8842            state.metadata.status_reason = None;
8843            state.metadata.exit_code = None;
8844            state.metadata.finished_at = None;
8845            state.metadata.duration_ms = None;
8846            crate::bash_background::persistence::write_task(&task.paths.json, &state.metadata)
8847                .expect("persist reset Running metadata");
8848            let zombie = spawn_unreaped_zombie();
8849            zombie_pid = zombie.id();
8850            state.runtime = TaskRuntime::Piped(Some(zombie));
8851        }
8852        *task.terminal_at.lock().unwrap() = None;
8853
8854        // Precondition: the installed child is genuinely a `<defunct>` zombie.
8855        assert!(
8856            is_zombie(zombie_pid),
8857            "precondition: stand-in child {zombie_pid} must be a zombie before finalize"
8858        );
8859
8860        // Drive the exit-marker terminal path. Before the fix this nulled the
8861        // Child handle without wait(), leaving the zombie behind.
8862        registry.poll_task(&task).unwrap();
8863
8864        {
8865            let state = task.state.lock().unwrap();
8866            assert!(
8867                matches!(state.runtime, TaskRuntime::Piped(None)),
8868                "child handle must be released after marker finalize"
8869            );
8870            assert!(
8871                state.metadata.status.is_terminal(),
8872                "task must be terminal after marker finalize: {:?}",
8873                state.metadata.status
8874            );
8875        }
8876
8877        // The core assertion: the child must have been REAPED, not just
8878        // dropped. A reaped PID has no `ps` row (or at minimum is not 'Z').
8879        assert!(
8880            !is_zombie(zombie_pid),
8881            "issue #91 regression: child {zombie_pid} left as <defunct> zombie \
8882             after the exit-marker terminal transition"
8883        );
8884    }
8885
8886    /// Companion to the above for the kill path: when a kill observes an
8887    /// already-present exit marker (the child finished on its own first), it
8888    /// must reap the child handle rather than dropping it.
8889    #[cfg(unix)]
8890    #[test]
8891    fn kill_with_existing_marker_reaps_child_no_zombie() {
8892        use std::sync::atomic::Ordering;
8893
8894        let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
8895        let dir = tempfile::tempdir().unwrap();
8896        let task_id = registry
8897            .spawn(
8898                SpawnPlan::Unsandboxed,
8899                QUICK_SUCCESS_COMMAND,
8900                "session".to_string(),
8901                dir.path().to_path_buf(),
8902                HashMap::new(),
8903                Some(Duration::from_secs(30)),
8904                dir.path().to_path_buf(),
8905                10,
8906                true,
8907                false,
8908                Some(dir.path().to_path_buf()),
8909            )
8910            .unwrap();
8911
8912        registry.inner.shutdown.store(true, Ordering::SeqCst);
8913        std::thread::sleep(Duration::from_millis(550));
8914
8915        let task = registry.task_for_session(&task_id, "session").unwrap();
8916
8917        let started = Instant::now();
8918        while !task.paths.exit.exists() {
8919            assert!(
8920                started.elapsed() < CHILD_EXIT_LIVENESS_BOUND,
8921                "exit marker should land quickly for `true`"
8922            );
8923            std::thread::sleep(Duration::from_millis(20));
8924        }
8925
8926        let zombie_pid;
8927        {
8928            let mut state = task.state.lock().unwrap();
8929            state.metadata.status = BgTaskStatus::Running;
8930            state.metadata.status_reason = None;
8931            state.metadata.exit_code = None;
8932            state.metadata.finished_at = None;
8933            state.metadata.duration_ms = None;
8934            crate::bash_background::persistence::write_task(&task.paths.json, &state.metadata)
8935                .expect("persist reset Running metadata");
8936            let zombie = spawn_unreaped_zombie();
8937            zombie_pid = zombie.id();
8938            state.runtime = TaskRuntime::Piped(Some(zombie));
8939        }
8940        *task.terminal_at.lock().unwrap() = None;
8941
8942        assert!(
8943            is_zombie(zombie_pid),
8944            "precondition: stand-in child {zombie_pid} must be a zombie before kill"
8945        );
8946
8947        // Kill observes the existing marker and finalizes from it.
8948        registry
8949            .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
8950            .expect("kill should succeed");
8951
8952        {
8953            let state = task.state.lock().unwrap();
8954            assert!(
8955                matches!(state.runtime, TaskRuntime::Piped(None)),
8956                "child handle must be released after marker-aware kill"
8957            );
8958            assert!(state.metadata.status.is_terminal());
8959        }
8960
8961        assert!(
8962            !is_zombie(zombie_pid),
8963            "issue #91 regression: child {zombie_pid} left as <defunct> zombie \
8964             after a marker-aware kill"
8965        );
8966    }
8967
8968    #[test]
8969    fn cleanup_finished_keeps_running_tasks() {
8970        let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
8971        let dir = tempfile::tempdir().unwrap();
8972        let task_id = registry
8973            .spawn(
8974                SpawnPlan::Unsandboxed,
8975                LONG_RUNNING_COMMAND,
8976                "session".to_string(),
8977                dir.path().to_path_buf(),
8978                HashMap::new(),
8979                Some(Duration::from_secs(30)),
8980                dir.path().to_path_buf(),
8981                10,
8982                true,
8983                false,
8984                Some(dir.path().to_path_buf()),
8985            )
8986            .unwrap();
8987
8988        registry.cleanup_finished(Duration::ZERO);
8989
8990        assert!(registry.inner.tasks.lock().unwrap().contains_key(&task_id));
8991        let _ = registry.kill(&task_id, "session");
8992    }
8993
8994    #[cfg(unix)]
8995    #[test]
8996    fn rehydrating_sandboxed_task_never_respawns_persisted_command() {
8997        let project = tempfile::tempdir().unwrap();
8998        let storage = tempfile::tempdir().unwrap();
8999        let sandbox_temp = storage.path().join("sandbox-temp");
9000        fs::create_dir(&sandbox_temp).unwrap();
9001        let launcher_script = project.path().join("sandbox-launch");
9002        let launcher = PathBuf::from("/bin/sh");
9003        fs::write(
9004            &launcher_script,
9005            "while [ \"$#\" -gt 0 ]; do\n  if [ \"$1\" = -- ]; then\n    shift\n    exec \"$@\"\n  fi\n  shift\ndone\nexit 78\n",
9006        )
9007        .unwrap();
9008        let mut permissions = fs::metadata(&launcher_script).unwrap().permissions();
9009        permissions.set_mode(0o700);
9010        fs::set_permissions(&launcher_script, permissions).unwrap();
9011
9012        let profile = crate::sandbox_profile::SandboxProfile::build(
9013            vec![project.path().to_path_buf()],
9014            Vec::new(),
9015            Vec::new(),
9016            Vec::new(),
9017            Vec::new(),
9018            Vec::new(),
9019            Vec::new(),
9020            sandbox_temp,
9021        )
9022        .unwrap();
9023        let plan = SpawnPlan::launcher_for_test(profile, launcher);
9024        let spawn_marker = project.path().join("spawn-count");
9025        let stop_marker = project.path().join("stop-command");
9026        let quote =
9027            |path: &Path| format!("'{}'", path.display().to_string().replace('\'', "'\\''"));
9028        let command = format!(
9029            "printf 'spawn\\n' >> {}; while [ ! -e {} ]; do sleep 0.05; done",
9030            quote(&spawn_marker),
9031            quote(&stop_marker)
9032        );
9033
9034        let original = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
9035        let task_id = original
9036            .spawn(
9037                plan,
9038                &command,
9039                "sandbox-rehydrate".to_string(),
9040                project.path().to_path_buf(),
9041                HashMap::new(),
9042                Some(Duration::from_secs(30)),
9043                storage.path().to_path_buf(),
9044                10,
9045                true,
9046                false,
9047                Some(project.path().to_path_buf()),
9048            )
9049            .unwrap();
9050        let started = Instant::now();
9051        while !spawn_marker.exists() {
9052            assert!(
9053                started.elapsed() < Duration::from_secs(20),
9054                "original sandboxed task did not start"
9055            );
9056            std::thread::sleep(Duration::from_millis(10));
9057        }
9058        original.detach();
9059
9060        let restarted = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
9061        restarted
9062            .replay_session(storage.path(), "sandbox-rehydrate")
9063            .unwrap();
9064        let replayed = restarted
9065            .status(
9066                &task_id,
9067                "sandbox-rehydrate",
9068                Some(project.path()),
9069                Some(storage.path()),
9070                4096,
9071            )
9072            .expect("rehydrated sandbox task");
9073        assert_eq!(replayed.info.status, BgTaskStatus::Running);
9074        assert!(replayed.sandbox_native);
9075
9076        std::thread::sleep(Duration::from_millis(650));
9077        assert_eq!(
9078            fs::read_to_string(&spawn_marker).unwrap().lines().count(),
9079            1,
9080            "registry replay must observe the persisted process without spawning its command"
9081        );
9082
9083        fs::write(&stop_marker, "stop").unwrap();
9084        let terminal = wait_for_terminal_snapshot(
9085            &restarted,
9086            &task_id,
9087            "sandbox-rehydrate",
9088            project.path(),
9089            storage.path(),
9090        );
9091        assert_eq!(terminal.info.status, BgTaskStatus::Completed);
9092        assert_eq!(
9093            fs::read_to_string(&spawn_marker).unwrap().lines().count(),
9094            1
9095        );
9096        restarted.detach();
9097    }
9098
9099    #[cfg(windows)]
9100    fn wait_for_file(path: &Path) -> String {
9101        // Task io/ artifacts are now pre-created empty (O_EXCL) at spawn under
9102        // the control/io split, then filled by the child (stdout/stderr) or the
9103        // daemon after observing exit (the exit marker). Existence is therefore
9104        // no longer a readiness signal — wait for non-empty content, matching
9105        // production's read_exit_marker, which treats an empty marker as "not
9106        // yet written".
9107        let started = Instant::now();
9108        loop {
9109            if let Ok(content) = fs::read_to_string(path) {
9110                if !content.trim().is_empty() {
9111                    return content;
9112                }
9113            }
9114            assert!(
9115                started.elapsed() < Duration::from_secs(30),
9116                "timed out waiting for non-empty {}",
9117                path.display()
9118            );
9119            std::thread::sleep(Duration::from_millis(100));
9120        }
9121    }
9122
9123    #[cfg(windows)]
9124    fn spawn_windows_registry_command(
9125        command: &str,
9126    ) -> (BgTaskRegistry, tempfile::TempDir, String) {
9127        let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
9128        let dir = tempfile::tempdir().unwrap();
9129        let task_id = registry
9130            .spawn(
9131                SpawnPlan::Unsandboxed,
9132                command,
9133                "session".to_string(),
9134                dir.path().to_path_buf(),
9135                HashMap::new(),
9136                Some(Duration::from_secs(30)),
9137                dir.path().to_path_buf(),
9138                10,
9139                false,
9140                false,
9141                Some(dir.path().to_path_buf()),
9142            )
9143            .unwrap();
9144        (registry, dir, task_id)
9145    }
9146
9147    #[cfg(windows)]
9148    #[test]
9149    fn windows_spawn_writes_exit_marker_for_zero_exit() {
9150        let (registry, _dir, task_id) = spawn_windows_registry_command("cmd /c exit 0");
9151        let exit_path = registry.task_exit_path(&task_id, "session").unwrap();
9152
9153        let content = wait_for_file(&exit_path);
9154
9155        assert_eq!(content.trim(), "0");
9156    }
9157
9158    #[cfg(windows)]
9159    #[test]
9160    fn windows_spawn_writes_exit_marker_for_nonzero_exit() {
9161        let (registry, _dir, task_id) = spawn_windows_registry_command("cmd /c exit 42");
9162        let exit_path = registry.task_exit_path(&task_id, "session").unwrap();
9163
9164        let content = wait_for_file(&exit_path);
9165
9166        assert_eq!(content.trim(), "42");
9167    }
9168
9169    #[cfg(windows)]
9170    #[test]
9171    fn windows_spawn_captures_stdout_to_disk() {
9172        let (registry, _dir, task_id) = spawn_windows_registry_command("cmd /c echo hello");
9173        let task = registry.task_for_session(&task_id, "session").unwrap();
9174        let stdout_path = task.paths.stdout.clone();
9175        let exit_path = task.paths.exit.clone();
9176
9177        let _ = wait_for_file(&exit_path);
9178        let stdout = fs::read_to_string(stdout_path).expect("read stdout");
9179
9180        assert!(stdout.contains("hello"), "stdout was {stdout:?}");
9181    }
9182
9183    #[cfg(windows)]
9184    #[test]
9185    fn windows_spawn_uses_pwsh_when_available() {
9186        // Without $SHELL set, $SHELL probe yields None and pwsh wins.
9187        // (We intentionally pass None for shell_env to keep this test
9188        // independent of the runner's actual env.)
9189        let candidates = crate::windows_shell::shell_candidates_with(
9190            |binary| match binary {
9191                "pwsh.exe" => Some(std::path::PathBuf::from(r"C:\pwsh\pwsh.exe")),
9192                "powershell.exe" => Some(std::path::PathBuf::from(r"C:\ps\powershell.exe")),
9193                _ => None,
9194            },
9195            || None,
9196        );
9197        let shell = candidates.first().expect("at least one candidate").clone();
9198        assert_eq!(shell, crate::windows_shell::WindowsShell::Pwsh);
9199        assert_eq!(shell.binary().as_ref(), "pwsh.exe");
9200    }
9201
9202    /// Windows wrappers return the command's status; the daemon writes the
9203    /// authoritative exit marker through its retained handle.
9204    #[cfg(windows)]
9205    #[test]
9206    fn windows_shell_cmd_wrapper_writes_marker_via_temp_rename() {
9207        let exit_path = Path::new(r"C:\Temp\bash-test.exit");
9208        let script =
9209            crate::windows_shell::WindowsShell::Cmd.wrapper_script("cmd /c exit 42", exit_path);
9210
9211        assert!(
9212            script.contains("set CODE=%ERRORLEVEL%"),
9213            "wrapper must capture the child exit code: {script}"
9214        );
9215        assert!(
9216            script.contains("exit /B %CODE%"),
9217            "wrapper must propagate the child exit code: {script}"
9218        );
9219        // The child records its own exit marker into io/ via temp-file +
9220        // rename so detached tasks whose spawning daemon is gone still report
9221        // exit. In-place writes are blocked by the daemon's retained io/exit
9222        // handle; the rename succeeds (FILE_SHARE_DELETE) and swaps atomically.
9223        assert!(
9224            script.contains("bash-test.exit"),
9225            "wrapper must target the exit marker path: {script}"
9226        );
9227        assert!(
9228            script.contains("move /Y"),
9229            "wrapper must write the marker atomically via temp-file + rename: {script}"
9230        );
9231    }
9232
9233    /// `bg_command()` for Cmd no longer needs `/V:ON` — the wrapper is now
9234    /// written to a `.bat` file where batch-line evaluation captures
9235    /// `%ERRORLEVEL%` correctly without delayed expansion. We still need
9236    /// `/D` (skip AutoRun) and `/S` (simple quote-stripping for paths with
9237    /// internal `"`-quoting from `cmd_quote`).
9238    #[cfg(windows)]
9239    #[test]
9240    fn windows_shell_cmd_bg_command_uses_minimal_cmd_flags() {
9241        use crate::windows_shell::WindowsShell;
9242        let cmd = WindowsShell::Cmd.bg_command("echo wrapped");
9243        let args: Vec<&std::ffi::OsStr> = cmd.get_args().collect();
9244        let args_strs: Vec<&str> = args.iter().filter_map(|a| a.to_str()).collect();
9245        assert_eq!(
9246            args_strs,
9247            vec!["/D", "/S", "/C", "echo wrapped"],
9248            "Cmd::bg_command must prepend /D /S /C"
9249        );
9250    }
9251
9252    /// PowerShell variants don't need `/V:ON`-style flags; their
9253    /// `bg_command()` args stay on the standard `-Command` path.
9254    #[cfg(windows)]
9255    #[test]
9256    fn windows_shell_pwsh_bg_command_uses_standard_args() {
9257        use crate::windows_shell::WindowsShell;
9258        let cmd = WindowsShell::Pwsh.bg_command("Get-Date");
9259        let args: Vec<&std::ffi::OsStr> = cmd.get_args().collect();
9260        let args_strs: Vec<&str> = args.iter().filter_map(|a| a.to_str()).collect();
9261        assert!(
9262            args_strs.contains(&"-Command"),
9263            "Pwsh::bg_command must use -Command: {args_strs:?}"
9264        );
9265        assert!(
9266            args_strs.contains(&"Get-Date"),
9267            "Pwsh::bg_command must include the user command body"
9268        );
9269    }
9270
9271    fn registry_with_db_and_frames(
9272        storage: &Path,
9273    ) -> (
9274        BgTaskRegistry,
9275        Arc<Mutex<TrackedConnection>>,
9276        Arc<Mutex<Vec<PushFrame>>>,
9277    ) {
9278        let frames = Arc::new(Mutex::new(Vec::new()));
9279        let captured = Arc::clone(&frames);
9280        let sender: crate::context::ProgressSender = Arc::new(Box::new(move |frame| {
9281            captured.lock().unwrap().push(frame);
9282        })
9283            as Box<dyn Fn(PushFrame) + Send + Sync>);
9284        let registry = BgTaskRegistry::new(Arc::new(Mutex::new(Some(sender))));
9285        registry.set_harness(Harness::Opencode);
9286        let conn = crate::db::open(&storage.join("aft.db")).expect("open test DB");
9287        let shared = Arc::new(Mutex::new(conn));
9288        registry.set_db_pool(shared.clone());
9289        (registry, shared, frames)
9290    }
9291
9292    fn pattern_match_frames(frames: &Mutex<Vec<PushFrame>>) -> Vec<BashPatternMatchFrame> {
9293        frames
9294            .lock()
9295            .unwrap()
9296            .iter()
9297            .filter_map(|frame| match frame {
9298                PushFrame::BashPatternMatch(frame) => Some(frame.clone()),
9299                _ => None,
9300            })
9301            .collect()
9302    }
9303
9304    fn install_delivered_terminal_with_pending_watch(
9305        registry: &BgTaskRegistry,
9306        db: &Arc<Mutex<TrackedConnection>>,
9307        storage: &Path,
9308        task_id: &str,
9309    ) -> TaskPaths {
9310        let paths = task_paths(storage, "session", task_id).unwrap();
9311        let mut metadata = PersistedTask::starting(
9312            task_id.to_string(),
9313            "session".to_string(),
9314            "false".to_string(),
9315            storage.to_path_buf(),
9316            Some(storage.to_path_buf()),
9317            None,
9318            true,
9319            true,
9320        );
9321        metadata.mark_terminal(BgTaskStatus::Failed, Some(1), None);
9322        metadata.completion_delivered = true;
9323        write_task(&paths.json, &metadata).unwrap();
9324        {
9325            let conn = db.lock().unwrap();
9326            crate::db::bash_tasks::upsert_bash_task(
9327                &conn,
9328                &metadata.to_bash_task_row("opencode", &paths).unwrap(),
9329            )
9330            .unwrap();
9331            crate::db::bash_watches::upsert_bash_pattern_watch(
9332                &conn,
9333                &BashPatternWatchRow {
9334                    harness: "opencode".into(),
9335                    session_id: "session".into(),
9336                    task_id: task_id.into(),
9337                    watch_id: "watch-00000001".into(),
9338                    pattern_kind: "substring".into(),
9339                    pattern: "(fail)".into(),
9340                    once: true,
9341                    created_at: 1,
9342                    stdout_offset: 756_243,
9343                    stderr_offset: 0,
9344                    pty_offset: 0,
9345                    scanning: false,
9346                    pending_match: true,
9347                    match_text: Some("(fail)".into()),
9348                    match_offset: Some(756_237),
9349                    match_context: Some("release output ... (fail)".into()),
9350                },
9351            )
9352            .unwrap();
9353        }
9354        registry
9355            .insert_rehydrated_task(metadata, paths.clone(), true)
9356            .unwrap();
9357        paths
9358    }
9359
9360    /// The window a concurrent spawn leaves open: the task directory exists but
9361    /// neither `control/` nor the metadata do yet. GC must leave it alone; an
9362    /// identical directory older than the grace is still quarantined, so the
9363    /// skip is age-bound rather than a blanket exemption.
9364    #[test]
9365    fn gc_skips_a_task_directory_still_being_created_but_quarantines_an_abandoned_one() {
9366        let dir = tempfile::tempdir().unwrap();
9367        let storage = dir.path();
9368        let registry = BgTaskRegistry::default();
9369        let session_dir = session_tasks_dir(storage, "session");
9370        let young = session_dir.join("bash-0000000000000301");
9371        let abandoned = session_dir.join("bash-0000000000000302");
9372        fs::create_dir_all(&young).unwrap();
9373        fs::create_dir_all(&abandoned).unwrap();
9374        let old = SystemTime::now() - Duration::from_secs(6 * 60);
9375        filetime::set_file_mtime(&abandoned, filetime::FileTime::from_system_time(old)).unwrap();
9376
9377        registry.maybe_gc_persisted(storage).unwrap();
9378
9379        assert!(
9380            young.is_dir(),
9381            "a task directory younger than the grace was quarantined mid-creation"
9382        );
9383        assert!(
9384            !abandoned.is_dir(),
9385            "an empty task directory older than the grace must still be quarantined"
9386        );
9387        let quarantined = fs::read_dir(storage.join("bash-tasks-quarantine"))
9388            .map(|entries| entries.flatten().count())
9389            .unwrap_or(0);
9390        assert_eq!(
9391            quarantined, 1,
9392            "exactly the abandoned layout is quarantined"
9393        );
9394    }
9395
9396    #[test]
9397    fn pending_pattern_match_is_returned_by_drain_contract_until_ack() {
9398        let storage = tempfile::tempdir().unwrap();
9399        let (registry, db, _frames) = registry_with_db_and_frames(storage.path());
9400        let task_id = "bash-0000000000000197";
9401        install_delivered_terminal_with_pending_watch(&registry, &db, storage.path(), task_id);
9402
9403        let first = registry.pending_pattern_matches_for_session("session");
9404        let second = registry.pending_pattern_matches_for_session("session");
9405        assert_eq!(first.len(), 1);
9406        assert_eq!(second.len(), 1);
9407        assert_eq!(first[0].task_id, task_id);
9408        assert_eq!(first[0].watch_id, "watch-00000001");
9409        assert_eq!(registry.unacked_wake_count_for_session(Some("session")), 1);
9410        let stuck =
9411            registry.stuck_pending_watches_for_session("session", Duration::from_secs(10 * 60));
9412        assert_eq!(stuck.len(), 1);
9413        assert_eq!(stuck[0].0, task_id);
9414        assert_eq!(stuck[0].1, "watch-00000001");
9415        assert!(stuck[0].2 >= Duration::from_secs(10 * 60).as_millis() as u64);
9416
9417        assert_eq!(
9418            registry.ack_completions_for_session(Some("session"), &[task_id.to_string()],),
9419            vec![task_id.to_string()]
9420        );
9421        assert!(registry
9422            .pending_pattern_matches_for_session("session")
9423            .is_empty());
9424        assert_eq!(registry.unacked_wake_count_for_session(Some("session")), 0);
9425    }
9426
9427    #[test]
9428    fn terminalized_watch_is_one_shot_across_registry_restart_after_ack() {
9429        let storage = tempfile::tempdir().unwrap();
9430        let (registry, db, _frames) = registry_with_db_and_frames(storage.path());
9431        let task_id = "bash-0000000000000196";
9432        install_delivered_terminal_with_pending_watch(&registry, &db, storage.path(), task_id);
9433        {
9434            let conn = db.lock().unwrap();
9435            let mut row = crate::db::bash_watches::get_bash_pattern_watch(
9436                &conn,
9437                "opencode",
9438                "session",
9439                task_id,
9440                "watch-00000001",
9441            )
9442            .unwrap()
9443            .unwrap();
9444            row.scanning = false;
9445            row.pending_match = true;
9446            row.match_text = Some(WATCH_TARGET_ERASED_TEXT.to_string());
9447            row.match_offset = Some(0);
9448            row.match_context = Some(WATCH_TARGET_ERASED_CONTEXT.to_string());
9449            crate::db::bash_watches::upsert_bash_pattern_watch(&conn, &row).unwrap();
9450        }
9451
9452        let terminalized = registry.pending_pattern_matches_for_session("session");
9453        assert_eq!(terminalized.len(), 1);
9454        assert_eq!(terminalized[0].task_id, task_id);
9455        assert_eq!(terminalized[0].match_text, WATCH_TARGET_ERASED_TEXT);
9456        assert_eq!(
9457            registry.ack_completions_for_session(Some("session"), &[task_id.to_string()]),
9458            vec![task_id.to_string()]
9459        );
9460        drop(registry);
9461
9462        let restarted = BgTaskRegistry::default();
9463        restarted.set_harness(Harness::Opencode);
9464        restarted.set_db_pool(db);
9465        assert!(restarted
9466            .pending_pattern_matches_for_session("session")
9467            .is_empty());
9468    }
9469
9470    #[test]
9471    fn standalone_same_project_multi_session_replay_preserves_earlier_session_delivery() {
9472        let storage = tempfile::tempdir().unwrap();
9473        let project = tempfile::tempdir().unwrap();
9474        let (registry, _db, frames) = registry_with_db_and_frames(storage.path());
9475
9476        // Session A replays for the project, spawns a watched task, and spawns a completion task.
9477        registry
9478            .replay_session_for_project(storage.path(), "session-a", project.path())
9479            .unwrap();
9480
9481        let task_watch = registry
9482            .spawn(
9483                SpawnPlan::Unsandboxed,
9484                LONG_RUNNING_COMMAND,
9485                "session-a".to_string(),
9486                project.path().to_path_buf(),
9487                HashMap::new(),
9488                Some(Duration::from_secs(30)),
9489                storage.path().to_path_buf(),
9490                10,
9491                true,
9492                false,
9493                Some(project.path().to_path_buf()),
9494            )
9495            .unwrap();
9496        registry
9497            .register_watch(
9498                task_watch.clone(),
9499                WatchPattern::Substring("READY".into()),
9500                true,
9501            )
9502            .unwrap();
9503
9504        let task_comp = registry
9505            .spawn(
9506                SpawnPlan::Unsandboxed,
9507                LONG_RUNNING_COMMAND,
9508                "session-a".to_string(),
9509                project.path().to_path_buf(),
9510                HashMap::new(),
9511                Some(Duration::from_secs(30)),
9512                storage.path().to_path_buf(),
9513                10,
9514                true,
9515                false,
9516                Some(project.path().to_path_buf()),
9517            )
9518            .unwrap();
9519
9520        // Session B replays for the same project on the shared standalone bridge.
9521        registry
9522            .replay_session_for_project(storage.path(), "session-b", project.path())
9523            .unwrap();
9524
9525        // Session A's watched task produces matching output.
9526        let watch_handle = registry.task_for_session(&task_watch, "session-a").unwrap();
9527        std::fs::OpenOptions::new()
9528            .append(true)
9529            .open(&watch_handle.paths.stdout)
9530            .unwrap()
9531            .write_all(b"READY\n")
9532            .unwrap();
9533        registry.scan_task_watch_output(&watch_handle);
9534
9535        // Session A's background completion task completes.
9536        let comp_handle = registry.task_for_session(&task_comp, "session-a").unwrap();
9537        fs::write(&comp_handle.paths.exit, "0\n").unwrap();
9538        comp_handle.mark_terminal_now();
9539        comp_handle.state.lock().unwrap().metadata.mark_terminal(
9540            BgTaskStatus::Completed,
9541            Some(0),
9542            None,
9543        );
9544        registry
9545            .post_terminal_transition(&comp_handle, true)
9546            .unwrap();
9547
9548        // The completion task was marked terminal while its child is still
9549        // running, so the descendant sampler finds a survivor and the frame is
9550        // emitted after its grace period rather than inline.
9551        let deadline = Instant::now() + Duration::from_secs(5);
9552        loop {
9553            let emitted = frames.lock().unwrap().iter().any(
9554                |frame| matches!(frame, PushFrame::BashCompleted(f) if f.task_id == task_comp),
9555            );
9556            if emitted || Instant::now() >= deadline {
9557                break;
9558            }
9559            std::thread::sleep(Duration::from_millis(20));
9560        }
9561
9562        let captured = frames.lock().unwrap();
9563        let pattern_matches = captured
9564            .iter()
9565            .filter_map(|frame| match frame {
9566                PushFrame::BashPatternMatch(frame) => Some(frame),
9567                _ => None,
9568            })
9569            .collect::<Vec<_>>();
9570        assert!(
9571            pattern_matches.iter().any(|f| f.task_id == task_watch
9572                && f.session_id == "session-a"
9573                && f.match_text == "READY"),
9574            "session A pattern match must be emitted after session B replay: {pattern_matches:?}"
9575        );
9576
9577        let completions = captured
9578            .iter()
9579            .filter_map(|frame| match frame {
9580                PushFrame::BashCompleted(frame) => Some(frame),
9581                _ => None,
9582            })
9583            .collect::<Vec<_>>();
9584        assert!(
9585            completions
9586                .iter()
9587                .any(|f| f.task_id == task_comp && f.session_id == "session-a"),
9588            "session A completion must be emitted after session B replay: {completions:?}"
9589        );
9590    }
9591
9592    #[cfg(unix)]
9593    #[test]
9594    fn gc_refuses_to_delete_or_quarantine_a_recorded_live_process() {
9595        let dir = tempfile::tempdir().unwrap();
9596        let storage = dir.path();
9597        let (registry, db, _frames) = registry_with_db_and_frames(storage);
9598        let task_id = "bash-0000000000000198";
9599        let paths = task_paths(storage, "session", task_id).unwrap();
9600        let mut running = PersistedTask::starting(
9601            task_id.to_string(),
9602            "session".to_string(),
9603            "live-process-canary".to_string(),
9604            storage.to_path_buf(),
9605            Some(storage.to_path_buf()),
9606            None,
9607            true,
9608            false,
9609        );
9610        // Own-PID is safe here because GC only CHECKS liveness (it refuses
9611        // to delete live bundles, never signals them). Kill-path tests must
9612        // spawn a disposable child instead - see bash_kill.rs.
9613        running.mark_running(std::process::id(), std::process::id() as i32);
9614        write_task(&paths.json, &running).unwrap();
9615        fs::write(&paths.stdout, b"").unwrap();
9616        fs::write(&paths.stderr, b"").unwrap();
9617        {
9618            let conn = db.lock().unwrap();
9619            crate::db::bash_tasks::upsert_bash_task(
9620                &conn,
9621                &running.to_bash_task_row("opencode", &paths).unwrap(),
9622            )
9623            .unwrap();
9624        }
9625        let running_json = fs::read(&paths.json).unwrap();
9626        let mut terminal: PersistedTask = serde_json::from_slice(&running_json).unwrap();
9627        terminal.mark_terminal(BgTaskStatus::Completed, Some(0), None);
9628        terminal.completion_delivered = true;
9629        write_task_at(
9630            &resolve_task_layout(&paths.session_dir, task_id).unwrap(),
9631            &terminal,
9632        )
9633        .unwrap();
9634        let old = SystemTime::now()
9635            .checked_sub(Duration::from_secs(25 * 60 * 60))
9636            .unwrap();
9637        filetime::set_file_mtime(&paths.json, filetime::FileTime::from_system_time(old)).unwrap();
9638
9639        assert_eq!(registry.maybe_gc_persisted(storage).unwrap(), 0);
9640        assert!(paths.io_dir.exists(), "GC deleted a live task bundle");
9641
9642        fs::write(&paths.json, b"{corrupt").unwrap();
9643        filetime::set_file_mtime(&paths.json, filetime::FileTime::from_system_time(old)).unwrap();
9644        assert_eq!(registry.maybe_gc_persisted(storage).unwrap(), 0);
9645        assert!(
9646            paths.io_dir.exists(),
9647            "GC quarantined a live task with unreadable metadata"
9648        );
9649
9650        fs::write(&paths.json, running_json).unwrap();
9651    }
9652
9653    #[test]
9654    fn pattern_watch_survives_registry_teardown_and_rehydrate() {
9655        let dir = tempfile::tempdir().unwrap();
9656        let storage = dir.path();
9657        let (registry, _db, frames) = registry_with_db_and_frames(storage);
9658        let task_id = registry
9659            .spawn(
9660                SpawnPlan::Unsandboxed,
9661                LONG_RUNNING_COMMAND,
9662                "session".to_string(),
9663                storage.to_path_buf(),
9664                HashMap::new(),
9665                Some(Duration::from_secs(30)),
9666                storage.to_path_buf(),
9667                10,
9668                true,
9669                false,
9670                Some(storage.to_path_buf()),
9671            )
9672            .unwrap();
9673        registry
9674            .register_watch(
9675                task_id.clone(),
9676                WatchPattern::Substring("READY".into()),
9677                true,
9678            )
9679            .unwrap();
9680        let task = registry.task_for_session(&task_id, "session").unwrap();
9681        // Simulate bridge death: drop in-memory watches only. Durable rows remain.
9682        registry.clear_task_watch_state(&task_id);
9683        assert_eq!(registry.active_watch_count(&task_id), 0);
9684
9685        std::fs::OpenOptions::new()
9686            .append(true)
9687            .open(&task.paths.stdout)
9688            .unwrap()
9689            .write_all(b"READY\n")
9690            .unwrap();
9691        frames.lock().unwrap().clear();
9692
9693        let (replayed, _db2, replay_frames) = registry_with_db_and_frames(storage);
9694        // Keep the original process's task row out of the way; replay loads from DB/disk.
9695        registry
9696            .inner
9697            .shutdown
9698            .store(true, std::sync::atomic::Ordering::SeqCst);
9699        replayed
9700            .replay_session_inner(storage, "session", None)
9701            .unwrap();
9702
9703        let matches = pattern_match_frames(&replay_frames);
9704        assert!(
9705            matches.iter().any(|frame| {
9706                frame.task_id == task_id
9707                    && frame.reason == "pattern_match"
9708                    && frame.match_text == "READY"
9709            }),
9710            "rehydrate should deliver gap match: {matches:?}"
9711        );
9712    }
9713
9714    #[test]
9715    fn pattern_watch_gap_match_between_teardown_and_rehydrate_delivers_once() {
9716        let dir = tempfile::tempdir().unwrap();
9717        let storage = dir.path();
9718        let (registry, _db, frames) = registry_with_db_and_frames(storage);
9719        let task_id = registry
9720            .spawn(
9721                SpawnPlan::Unsandboxed,
9722                LONG_RUNNING_COMMAND,
9723                "session".to_string(),
9724                storage.to_path_buf(),
9725                HashMap::new(),
9726                Some(Duration::from_secs(30)),
9727                storage.to_path_buf(),
9728                10,
9729                true,
9730                false,
9731                Some(storage.to_path_buf()),
9732            )
9733            .unwrap();
9734        registry
9735            .register_watch(
9736                task_id.clone(),
9737                WatchPattern::Substring("GAP-HIT".into()),
9738                true,
9739            )
9740            .unwrap();
9741        let task = registry.task_for_session(&task_id, "session").unwrap();
9742        let cursor_before = registry.watch_stream_cursors(&task_id).0;
9743        registry.clear_task_watch_state(&task_id);
9744
9745        // Bytes land while the watch registry is down.
9746        std::fs::OpenOptions::new()
9747            .append(true)
9748            .open(&task.paths.stdout)
9749            .unwrap()
9750            .write_all(b"prefix GAP-HIT suffix\n")
9751            .unwrap();
9752        frames.lock().unwrap().clear();
9753
9754        let (replayed, _db2, replay_frames) = registry_with_db_and_frames(storage);
9755        registry
9756            .inner
9757            .shutdown
9758            .store(true, std::sync::atomic::Ordering::SeqCst);
9759        replayed
9760            .replay_session_inner(storage, "session", None)
9761            .unwrap();
9762
9763        let matches: Vec<_> = pattern_match_frames(&replay_frames)
9764            .into_iter()
9765            .filter(|frame| frame.task_id == task_id && frame.match_text.contains("GAP-HIT"))
9766            .collect();
9767        assert_eq!(
9768            matches.len(),
9769            1,
9770            "gap match must deliver exactly once: {matches:?}"
9771        );
9772        assert!(
9773            matches[0].match_offset >= cursor_before,
9774            "match offset should be at/after the persisted cursor ({cursor_before}), got {}",
9775            matches[0].match_offset
9776        );
9777    }
9778
9779    #[test]
9780    fn pattern_watch_acked_match_does_not_redeliver_after_restart() {
9781        let dir = tempfile::tempdir().unwrap();
9782        let storage = dir.path();
9783        let (registry, db, frames) = registry_with_db_and_frames(storage);
9784        let task_id = registry
9785            .spawn(
9786                SpawnPlan::Unsandboxed,
9787                LONG_RUNNING_COMMAND,
9788                "session".to_string(),
9789                storage.to_path_buf(),
9790                HashMap::new(),
9791                Some(Duration::from_secs(30)),
9792                storage.to_path_buf(),
9793                10,
9794                true,
9795                false,
9796                Some(storage.to_path_buf()),
9797            )
9798            .unwrap();
9799        registry
9800            .register_watch(
9801                task_id.clone(),
9802                WatchPattern::Substring("READY".into()),
9803                true,
9804            )
9805            .unwrap();
9806        let task = registry.task_for_session(&task_id, "session").unwrap();
9807        std::fs::OpenOptions::new()
9808            .append(true)
9809            .open(&task.paths.stdout)
9810            .unwrap()
9811            .write_all(b"READY\n")
9812            .unwrap();
9813        registry.scan_task_watch_output(&task);
9814        let delivered = pattern_match_frames(&frames);
9815        assert!(
9816            delivered
9817                .iter()
9818                .any(|frame| frame.task_id == task_id && frame.match_text == "READY"),
9819            "live path should deliver match: {delivered:?}"
9820        );
9821        // Ack via the same lane completions use.
9822        assert!(registry
9823            .ack_completions_for_session(Some("session"), std::slice::from_ref(&task_id))
9824            .contains(&task_id));
9825        {
9826            let conn = db.lock().unwrap();
9827            let rows = crate::db::bash_watches::list_bash_pattern_watches_for_task(
9828                &conn, "opencode", "session", &task_id,
9829            )
9830            .unwrap();
9831            assert!(
9832                rows.is_empty(),
9833                "acked once-watch rows must be deleted: {rows:?}"
9834            );
9835        }
9836
9837        frames.lock().unwrap().clear();
9838        let (replayed, _db2, replay_frames) = registry_with_db_and_frames(storage);
9839        registry
9840            .inner
9841            .shutdown
9842            .store(true, std::sync::atomic::Ordering::SeqCst);
9843        replayed
9844            .replay_session_inner(storage, "session", None)
9845            .unwrap();
9846        let matches = pattern_match_frames(&replay_frames)
9847            .into_iter()
9848            .filter(|frame| frame.task_id == task_id)
9849            .collect::<Vec<_>>();
9850        assert!(
9851            matches.is_empty(),
9852            "acked match must not re-deliver after restart: {matches:?}"
9853        );
9854    }
9855
9856    #[test]
9857    fn pending_watch_match_redelivers_after_terminal_cleanup_removes_task_bundle() {
9858        let dir = tempfile::tempdir().unwrap();
9859        let storage = dir.path();
9860        let (registry, db, frames) = registry_with_db_and_frames(storage);
9861        let task_id = "bash-aaaaaaaaaaaaaaa1";
9862        let paths = install_delivered_terminal_with_pending_watch(&registry, &db, storage, task_id);
9863        frames.lock().unwrap().clear();
9864
9865        registry.cleanup_finished(Duration::ZERO);
9866        assert!(registry.task(task_id).is_none());
9867        assert!(
9868            !paths.json.exists(),
9869            "cleanup must remove the task metadata bundle"
9870        );
9871
9872        let _ = registry.drain_completions_for_session(Some("session"));
9873        let matches = pattern_match_frames(&frames);
9874        assert!(
9875            matches.iter().any(|frame| {
9876                frame.task_id == task_id
9877                    && frame.watch_id == "watch-00000001"
9878                    && frame.match_text == "(fail)"
9879            }),
9880            "durable pending match must redeliver without an in-memory task: {matches:?}"
9881        );
9882        assert!(registry
9883            .ack_completions_for_session(Some("session"), &[task_id.to_string()])
9884            .contains(&task_id.to_string()));
9885        let rows = crate::db::bash_watches::list_bash_pattern_watches_for_task(
9886            &db.lock().unwrap(),
9887            "opencode",
9888            "session",
9889            task_id,
9890        )
9891        .unwrap();
9892        assert!(rows.is_empty(), "ack must end durable redelivery");
9893    }
9894
9895    #[test]
9896    fn status_uses_intact_terminal_db_row_after_task_bundle_cleanup() {
9897        let dir = tempfile::tempdir().unwrap();
9898        let storage = dir.path();
9899        let (registry, db, _frames) = registry_with_db_and_frames(storage);
9900        let task_id = "bash-aaaaaaaaaaaaaaa2";
9901        let paths = install_delivered_terminal_with_pending_watch(&registry, &db, storage, task_id);
9902
9903        registry.cleanup_finished(Duration::ZERO);
9904        assert!(registry.task(task_id).is_none());
9905        assert!(
9906            !paths.json.exists(),
9907            "cleanup must remove the task metadata bundle"
9908        );
9909        assert!(
9910            crate::db::bash_tasks::get_bash_task(
9911                &db.lock().unwrap(),
9912                "opencode",
9913                "session",
9914                task_id
9915            )
9916            .unwrap()
9917            .is_some(),
9918            "cleanup must retain the terminal database row"
9919        );
9920
9921        let snapshot = registry
9922            .status(
9923                task_id,
9924                "session",
9925                Some(storage),
9926                Some(storage),
9927                RUNNING_OUTPUT_PREVIEW_BYTES,
9928            )
9929            .expect("intact terminal row must remain visible after artifact cleanup");
9930        assert_eq!(snapshot.info.status, BgTaskStatus::Failed);
9931        assert_eq!(snapshot.exit_code, Some(1));
9932        assert!(snapshot.info.duration_ms.is_some());
9933    }
9934
9935    #[test]
9936    fn pattern_watch_rows_are_deleted_when_task_is_gc_deleted() {
9937        let dir = tempfile::tempdir().unwrap();
9938        let storage = dir.path();
9939        let (registry, db, _frames) = registry_with_db_and_frames(storage);
9940        let task_id = "bash-aaaaaaaaaaaaaaaa";
9941        let paths = task_paths(storage, "session", task_id).unwrap();
9942        let mut metadata = PersistedTask::starting(
9943            task_id.to_string(),
9944            "session".to_string(),
9945            "true".to_string(),
9946            storage.to_path_buf(),
9947            Some(storage.to_path_buf()),
9948            None,
9949            true,
9950            true,
9951        );
9952        metadata.mark_terminal(BgTaskStatus::Completed, Some(0), None);
9953        metadata.completion_delivered = true;
9954        write_task(&paths.json, &metadata).unwrap();
9955        {
9956            let conn = db.lock().unwrap();
9957            crate::db::bash_tasks::upsert_bash_task(
9958                &conn,
9959                &metadata.to_bash_task_row("opencode", &paths).unwrap(),
9960            )
9961            .unwrap();
9962            crate::db::bash_watches::upsert_bash_pattern_watch(
9963                &conn,
9964                &BashPatternWatchRow {
9965                    harness: "opencode".into(),
9966                    session_id: "session".into(),
9967                    task_id: task_id.into(),
9968                    watch_id: "watch-00000001".into(),
9969                    pattern_kind: "substring".into(),
9970                    pattern: "x".into(),
9971                    once: true,
9972                    created_at: 1,
9973                    stdout_offset: 0,
9974                    stderr_offset: 0,
9975                    pty_offset: 0,
9976                    scanning: true,
9977                    pending_match: false,
9978                    match_text: None,
9979                    match_offset: None,
9980                    match_context: None,
9981                },
9982            )
9983            .unwrap();
9984        }
9985        let old = SystemTime::now()
9986            .checked_sub(Duration::from_secs(25 * 60 * 60))
9987            .unwrap();
9988        filetime::set_file_mtime(&paths.json, filetime::FileTime::from_system_time(old)).unwrap();
9989
9990        let deleted = registry.maybe_gc_persisted(storage).unwrap();
9991        assert!(
9992            deleted >= 1,
9993            "expected GC to delete the terminal task bundle"
9994        );
9995        let conn = db.lock().unwrap();
9996        let watches = crate::db::bash_watches::list_bash_pattern_watches_for_task(
9997            &conn, "opencode", "session", task_id,
9998        )
9999        .unwrap();
10000        assert!(
10001            watches.is_empty(),
10002            "task-row GC must cascade to every persisted watch"
10003        );
10004    }
10005}