1use std::collections::{BTreeMap, HashMap, HashSet, VecDeque};
2#[cfg(not(windows))]
3use std::ffi::OsString;
4use std::fs;
5use std::io::Write;
6use std::path::{Path, PathBuf};
7#[cfg(windows)]
8use std::process::Command;
9use std::process::{Child, Stdio};
10use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
11#[cfg(unix)]
12use std::sync::OnceLock;
13use std::sync::{Arc, Mutex, RwLock};
14use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
15
16use crate::db::TrackedConnection;
17use serde::Serialize;
18
19use crate::bash_permissions::PermissionAsk;
20use crate::compress::caps::DropClass;
21#[cfg(unix)]
22use crate::compress::single_top_level_pipeline;
23use crate::compress::CompressionResult;
24use crate::context::SharedProgressSender;
25use crate::db::compression_events::CompressionAggregateCache;
26use crate::harness::Harness;
27use crate::list_envelope::ListEnvelope;
28use crate::protocol::{BashCompletedFrame, BashLongRunningFrame, BashPatternMatchFrame, PushFrame};
29use crate::sandbox_spawn::SpawnPlan;
30
31#[cfg(windows)]
32use std::os::windows::process::CommandExt;
33
34use super::buffer::{combine_streams, BgBuffer, DiskTruncation, StreamKind, TokenCountInput};
35use super::output::{
36 cap_completion_output, cap_completion_output_with_marker, cap_final_output,
37 cap_final_output_with_marker, completion_preview_threshold, json_output_pointer, quote_path,
38 retained_json_output_pointer, COMPRESS_INPUT_CAP_BYTES, COMPRESS_INPUT_HEAD_BYTES,
39 COMPRESS_INPUT_TAIL_BYTES, FINAL_OUTPUT_CAP_BYTES, RAW_PASSTHROUGH_CAP_BYTES,
40 RAW_PASSTHROUGH_HEAD_BYTES, RAW_PASSTHROUGH_TAIL_BYTES, RUNNING_OUTPUT_PREVIEW_BYTES,
41 STRUCTURED_OUTPUT_CAP_BYTES,
42};
43use super::persistence::{
44 allocate_task_layout, delete_resolved_task, delete_task_bundle, discover_task_ids,
45 open_task_artifact, quarantine_invalid_entry, quarantine_task_layout, read_exit_marker,
46 read_task_at, resolve_task_layout, session_tasks_dir, uninitialized_layout_is_recent,
47 unix_millis, update_task_at, validate_task_id, write_kill_marker_if_absent, write_task_at,
48 BgMode, ExitMarker, PersistedTask, TaskArtifact, TaskIoHandles, TaskPaths,
49};
50#[cfg(unix)]
51use super::process::terminate_pgid;
52#[cfg(windows)]
53use super::process::terminate_pid;
54use super::process::{
55 is_process_alive, is_recorded_process_alive, live_process_group_members, LiveDescendant,
56};
57use super::pty_process::spawn_pty_for_command;
58use super::pty_runtime::PtyRuntime;
59use super::watches::{
60 PatternMatch, WatchPattern, WatchRegistry, WATCH_TARGET_ERASED_CONTEXT,
61 WATCH_TARGET_ERASED_TEXT, WATCH_TASK_EXIT_TEXT,
62};
63use super::{BgTaskInfo, BgTaskStatus};
64use crate::db::bash_tasks::BashTaskRow;
65use crate::db::bash_watches::BashPatternWatchRow;
66const DEFAULT_BG_TIMEOUT: Duration = Duration::from_secs(30 * 60);
69const PERSISTED_GC_GRACE: Duration = Duration::from_secs(24 * 60 * 60);
70const QUARANTINE_GC_GRACE: Duration = Duration::from_secs(30 * 24 * 60 * 60);
71
72const TOKENIZE_CAP_BYTES_PER_STREAM: usize = 128 * 1024;
73pub const ROOT_RECLAIMED_REASON: &str = "root_reclaimed";
74#[cfg(target_os = "linux")]
75pub(crate) const LINUX_SCOPE_ENV: &str = "AFT_INTERNAL_LINUX_SCOPE";
76
77#[derive(Debug, Clone, Serialize)]
78pub struct BgCompletion {
79 pub task_id: String,
80 #[serde(skip_serializing)]
83 pub session_id: String,
84 pub status: BgTaskStatus,
85 pub exit_code: Option<i32>,
86 pub command: String,
87 #[serde(default, skip_serializing_if = "String::is_empty")]
93 pub output_preview: String,
94 #[serde(default, skip_serializing_if = "Option::is_none")]
95 pub bash_output_list_envelope: Option<ListEnvelope>,
96 #[serde(default, skip_serializing_if = "is_false")]
101 pub output_truncated: bool,
102 #[serde(default, skip_serializing_if = "Option::is_none")]
105 pub original_tokens: Option<u32>,
106 #[serde(default, skip_serializing_if = "Option::is_none")]
109 pub compressed_tokens: Option<u32>,
110 #[serde(default, skip_serializing_if = "is_false")]
112 pub tokens_skipped: bool,
113 #[serde(default, skip_serializing_if = "Option::is_none")]
114 pub status_reason: Option<String>,
115 pub live_descendants: Option<Vec<LiveDescendant>>,
116 #[serde(default, skip_serializing_if = "is_zero_usize")]
117 pub live_descendants_omitted: usize,
118 #[serde(default, skip_serializing_if = "Option::is_none")]
119 pub live_descendants_summary: Option<String>,
120}
121
122fn is_zero_usize(value: &usize) -> bool {
123 *value == 0
124}
125
126fn is_false(v: &bool) -> bool {
127 !*v
128}
129
130#[derive(Debug, Clone, Serialize)]
131pub struct BgTaskSnapshot {
132 #[serde(flatten)]
133 pub info: BgTaskInfo,
134 pub exit_code: Option<i32>,
135 pub child_pid: Option<u32>,
136 pub workdir: String,
137 pub output_preview: String,
138 #[serde(default, skip_serializing_if = "Option::is_none")]
139 pub bash_output_list_envelope: Option<ListEnvelope>,
140 pub output_truncated: bool,
141 pub output_path: Option<String>,
142 pub stderr_path: Option<String>,
143 #[serde(skip_serializing_if = "Option::is_none")]
144 pub pty_rows: Option<u16>,
145 #[serde(skip_serializing_if = "Option::is_none")]
146 pub pty_cols: Option<u16>,
147 #[serde(skip_serializing_if = "Option::is_none")]
148 pub pty_screen: Option<String>,
149 #[serde(default, skip_serializing_if = "Vec::is_empty")]
150 pub scanner_report: Vec<PermissionAsk>,
151 #[serde(default, skip_serializing_if = "is_false")]
152 pub sandbox_native: bool,
153 #[serde(default, skip_serializing_if = "is_false")]
154 pub sandbox_unavailable: bool,
155 pub live_descendants: Option<Vec<LiveDescendant>>,
156 #[serde(default, skip_serializing_if = "is_zero_usize")]
157 pub live_descendants_omitted: usize,
158 #[serde(default, skip_serializing_if = "Option::is_none")]
159 pub live_descendants_summary: Option<String>,
160 #[serde(default, skip_serializing_if = "is_false")]
161 pub kill_signaled: bool,
162 #[serde(default, skip_serializing_if = "is_zero_usize")]
163 pub kill_reached: usize,
164}
165
166#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
167pub struct BgTaskHealthCounts {
168 pub running: usize,
169 pub pending_completions: usize,
170}
171
172#[derive(Debug, Clone, Copy, PartialEq, Eq)]
173enum TerminalOutputKind {
174 Compressed,
175 Raw,
176 Structured,
177}
178
179#[derive(Debug, Clone, PartialEq, Eq)]
180struct TerminalOutputCache {
181 output_preview: String,
182 output_truncated: bool,
183 compression_input_line_count: Option<usize>,
184 kind: TerminalOutputKind,
185 output_path: Option<String>,
186 stderr_path: Option<String>,
187 artifact_access: ArtifactRecoveryAccess,
188 recovery: Option<RecoveryContext>,
189}
190
191#[derive(Debug, Clone, PartialEq, Eq)]
192struct ArtifactRecoveryAccess {
193 task_id: String,
194 readable: bool,
195}
196
197#[derive(Debug, Clone, PartialEq, Eq)]
198struct RecoveryContext {
199 dropped_by_class: BTreeMap<DropClass, usize>,
200 had_inner_drop: bool,
201 offset_hint_eligible: bool,
202 offset_start_line: Option<usize>,
203 byte_truncated: bool,
204 disk_truncated_prefix_bytes: u64,
205 output_path: Option<String>,
206 stderr_path: Option<String>,
207 include_stderr_path: bool,
208 artifact_access: ArtifactRecoveryAccess,
209}
210
211fn optional_string_bytes(value: Option<&String>) -> u64 {
212 value
213 .map(|value| crate::memory::usize_to_u64(value.len()))
214 .unwrap_or(0)
215}
216
217fn terminal_output_cache_estimated_bytes(cache: &TerminalOutputCache) -> u64 {
218 let recovery_bytes = cache
219 .recovery
220 .as_ref()
221 .map(|recovery| {
222 crate::memory::usize_to_u64(recovery.dropped_by_class.len())
223 .saturating_mul(
224 (std::mem::size_of::<DropClass>() + std::mem::size_of::<usize>()) as u64,
225 )
226 .saturating_add(optional_string_bytes(recovery.output_path.as_ref()))
227 .saturating_add(optional_string_bytes(recovery.stderr_path.as_ref()))
228 .saturating_add(crate::memory::usize_to_u64(
229 recovery.artifact_access.task_id.len(),
230 ))
231 })
232 .unwrap_or(0);
233 (std::mem::size_of::<TerminalOutputCache>() as u64)
234 .saturating_add(crate::memory::usize_to_u64(cache.output_preview.len()))
235 .saturating_add(optional_string_bytes(cache.output_path.as_ref()))
236 .saturating_add(optional_string_bytes(cache.stderr_path.as_ref()))
237 .saturating_add(crate::memory::usize_to_u64(
238 cache.artifact_access.task_id.len(),
239 ))
240 .saturating_add(recovery_bytes)
241}
242
243fn completion_estimated_bytes(completion: &BgCompletion) -> u64 {
244 (std::mem::size_of::<BgCompletion>() as u64)
245 .saturating_add(crate::memory::usize_to_u64(completion.task_id.len()))
246 .saturating_add(crate::memory::usize_to_u64(completion.session_id.len()))
247 .saturating_add(crate::memory::usize_to_u64(completion.command.len()))
248 .saturating_add(crate::memory::usize_to_u64(completion.output_preview.len()))
249}
250
251impl RecoveryContext {
252 fn has_visible_drop(&self) -> bool {
253 self.byte_truncated
254 || self.disk_truncated_prefix_bytes > 0
255 || self.had_inner_drop
256 || !self.dropped_by_class.is_empty()
257 }
258}
259
260#[derive(Clone)]
261pub struct BgTaskRegistry {
262 pub(crate) inner: Arc<RegistryInner>,
263}
264
265#[derive(Debug, Clone, Copy, PartialEq, Eq)]
267pub enum WatchdogPassCause {
268 Wake,
270 Tick,
272}
273
274pub(crate) struct RegistryInner {
275 pub(crate) tasks: Mutex<HashMap<String, Arc<BgTask>>>,
276 pub(crate) completions: Mutex<VecDeque<BgCompletion>>,
277 pub(crate) progress_sender: SharedProgressSender,
278 watchdog_started: AtomicBool,
279 pub(crate) shutdown: AtomicBool,
280 pub(crate) long_running_reminder_enabled: AtomicBool,
281 pub(crate) long_running_reminder_interval_ms: AtomicU64,
282 persisted_gc_started: AtomicBool,
283 #[cfg(test)]
284 persisted_gc_runs: AtomicU64,
285 persisted_gc_thread: Mutex<Option<String>>,
289 pub(crate) compressor:
295 Mutex<Option<Box<dyn Fn(&str, String, Option<i32>) -> CompressionResult + Send + Sync>>>,
296 pub(crate) db_pool: RwLock<Option<Arc<Mutex<TrackedConnection>>>>,
297 pub(crate) db_harness: RwLock<Option<String>>,
298 pub(crate) compression_aggregates: Arc<CompressionAggregateCache>,
299 pub(crate) wake_tx: crossbeam_channel::Sender<()>,
300 pub(crate) wake_rx: crossbeam_channel::Receiver<()>,
301 terminal_transition: tokio::sync::Notify,
302 pub(crate) completion_pass_cause: Mutex<HashMap<String, WatchdogPassCause>>,
306 pub(crate) watch_registry: Mutex<WatchRegistry>,
307 live_delivery_sessions: Mutex<HashSet<String>>,
312 wait_detach_sessions: Mutex<HashSet<String>>,
313 active_wait_sessions: Mutex<HashMap<String, usize>>,
314 wait_registered_tasks: Mutex<HashMap<String, HashSet<String>>>,
315}
316
317pub(crate) struct BgTask {
318 pub(crate) task_id: String,
319 pub(crate) session_id: String,
320 pub(crate) paths: TaskPaths,
321 artifact_root: PathBuf,
322 pub(crate) started: Instant,
323 pub(crate) last_reminder_at: Mutex<Option<Instant>>,
324 pub(crate) terminal_at: Mutex<Option<Instant>>,
325 pub(crate) state: Mutex<BgTaskState>,
326}
327
328pub(crate) enum TaskRuntime {
329 Piped(Option<Child>),
330 Pty(Option<PtyRuntime>),
331}
332
333pub(crate) struct BgTaskState {
334 pub(crate) metadata: PersistedTask,
335 pub(crate) runtime: TaskRuntime,
336 pub(crate) io_handles: Option<TaskIoHandles>,
339 pub(crate) detached: bool,
340 pub(crate) child_exit_observed: bool,
351 pub(crate) descendant_sampling_started: bool,
354 pub(crate) buffer: BgBuffer,
355 terminal_output_cache: Option<TerminalOutputCache>,
356 pub(crate) pending_terminal_override: Option<BgTaskStatus>,
358}
359
360fn completion_matches_session(completion: &BgCompletion, session_id: Option<&str>) -> bool {
361 session_id
362 .map(|session_id| completion.session_id == session_id)
363 .unwrap_or(true)
364}
365
366impl BgTaskRegistry {
367 pub fn completion_pass_cause(&self, task_id: &str) -> Option<WatchdogPassCause> {
370 self.inner
371 .completion_pass_cause
372 .lock()
373 .ok()
374 .and_then(|map| map.get(task_id).copied())
375 }
376
377 pub fn new(progress_sender: SharedProgressSender) -> Self {
378 let (wake_tx, wake_rx) = crossbeam_channel::bounded(1);
379 Self {
380 inner: Arc::new(RegistryInner {
381 tasks: Mutex::new(HashMap::new()),
382 completions: Mutex::new(VecDeque::new()),
383 progress_sender,
384 watchdog_started: AtomicBool::new(false),
385 shutdown: AtomicBool::new(false),
386 long_running_reminder_enabled: AtomicBool::new(true),
387 long_running_reminder_interval_ms: AtomicU64::new(600_000),
388 persisted_gc_started: AtomicBool::new(false),
389 #[cfg(test)]
390 persisted_gc_runs: AtomicU64::new(0),
391 persisted_gc_thread: Mutex::new(None),
392 compressor: Mutex::new(None),
393 db_pool: RwLock::new(None),
394 db_harness: RwLock::new(None),
395 compression_aggregates: Arc::new(CompressionAggregateCache::default()),
396 wake_tx,
397 wake_rx,
398 terminal_transition: tokio::sync::Notify::new(),
399 completion_pass_cause: Mutex::new(HashMap::new()),
400 watch_registry: Mutex::new(WatchRegistry::default()),
401 live_delivery_sessions: Mutex::new(HashSet::new()),
402 wait_detach_sessions: Mutex::new(HashSet::new()),
403 active_wait_sessions: Mutex::new(HashMap::new()),
404 wait_registered_tasks: Mutex::new(HashMap::new()),
405 }),
406 }
407 }
408
409 pub fn is_session_owned_artifact_path(&self, session_id: &str, path: &Path) -> bool {
417 let Ok(requested) = fs::canonicalize(path) else {
418 return false;
419 };
420 let Ok(tasks) = self.inner.tasks.lock() else {
421 return false;
422 };
423
424 tasks.values().any(|task| {
425 task.session_id == session_id
426 && [
427 &task.paths.stdout,
428 &task.paths.stderr,
429 &task.paths.exit,
430 &task.paths.pty,
431 ]
432 .into_iter()
433 .filter_map(|known| known.file_name())
434 .any(|name| task.artifact_root.join(name) == requested)
435 })
436 }
437
438 pub fn read_artifact_path(
439 &self,
440 session_id: &str,
441 path: &Path,
442 ) -> Option<Result<Vec<u8>, String>> {
443 let requested = fs::canonicalize(path).ok()?;
444 let tasks = self.inner.tasks.lock().ok()?;
445 let (task, artifact) = tasks.values().find_map(|task| {
446 if task.session_id != session_id {
447 return None;
448 }
449 TaskArtifact::ALL.into_iter().find_map(|artifact| {
450 let expected = task
451 .paths
452 .artifact_path(artifact)
453 .file_name()
454 .map(|name| task.artifact_root.join(name));
455 (expected.as_deref() == Some(requested.as_path()))
456 .then(|| (Arc::clone(task), artifact))
457 })
458 })?;
459 drop(tasks);
460 Some(self.read_artifact(&task.task_id, session_id, artifact))
461 }
462
463 pub fn read_artifact_range(
464 &self,
465 task_id: &str,
466 session_id: &str,
467 artifact: TaskArtifact,
468 offset: u64,
469 ) -> Result<(Vec<u8>, u64), String> {
470 validate_task_id(task_id).map_err(|error| error.to_string())?;
471 let task = self
472 .task_for_session(task_id, session_id)
473 .ok_or_else(|| "task_not_found".to_string())?;
474 let mut file = open_task_artifact(&task.paths, artifact)
475 .map_err(|error| format!("artifact_refused: {error}"))?;
476 let len = file
477 .len()
478 .map_err(|error| format!("artifact_refused: {error}"))?;
479 let start = offset.min(len);
480 let bytes = file
481 .read_range(start, len.saturating_sub(start))
482 .map_err(|error| format!("artifact_refused: {error}"))?;
483 Ok((bytes, len))
484 }
485
486 pub fn read_artifact(
487 &self,
488 task_id: &str,
489 session_id: &str,
490 artifact: TaskArtifact,
491 ) -> Result<Vec<u8>, String> {
492 validate_task_id(task_id).map_err(|error| error.to_string())?;
493 let task = self
494 .task_for_session(task_id, session_id)
495 .ok_or_else(|| "task_not_found".to_string())?;
496 let mut file = open_task_artifact(&task.paths, artifact)
497 .map_err(|error| format!("artifact_refused: {error}"))?;
498 file.read_all()
499 .map_err(|error| format!("artifact_refused: {error}"))
500 }
501
502 pub fn set_harness(&self, harness: Harness) {
503 if let Ok(mut slot) = self.inner.db_harness.write() {
504 *slot = Some(harness.storage_segment());
505 }
506 }
507
508 pub fn set_db_pool(&self, conn: Arc<Mutex<TrackedConnection>>) {
509 if let Ok(mut slot) = self.inner.db_pool.write() {
510 *slot = Some(conn);
511 }
512 self.inner.compression_aggregates.clear();
513 }
514
515 pub fn clear_db_pool(&self) {
516 if let Ok(mut slot) = self.inner.db_pool.write() {
517 *slot = None;
518 }
519 self.inner.compression_aggregates.clear();
520 }
521
522 pub(crate) fn record_live_delivery_session(&self, session_id: &str) {
523 if let Ok(mut live_sessions) = self.inner.live_delivery_sessions.lock() {
524 live_sessions.insert(session_id.to_string());
525 }
526 }
527
528 pub(crate) fn replace_live_delivery_sessions(&self, sessions: HashSet<String>) {
529 if let Ok(mut live_sessions) = self.inner.live_delivery_sessions.lock() {
530 *live_sessions = sessions;
531 }
532 }
533
534 fn originating_session_has_live_route(&self, session_id: &str) -> bool {
535 self.inner
536 .live_delivery_sessions
537 .lock()
538 .map(|sessions| sessions.contains(session_id))
539 .unwrap_or(true)
540 }
541
542 fn should_retire_foreign_delivery(
543 &self,
544 originating_session_id: &str,
545 binding_session_id: &str,
546 ) -> bool {
547 originating_session_id != binding_session_id
548 && !self.originating_session_has_live_route(originating_session_id)
549 }
550
551 pub(crate) fn compression_aggregate_cache(&self) -> Arc<CompressionAggregateCache> {
552 Arc::clone(&self.inner.compression_aggregates)
553 }
554
555 pub fn register_foreground_task(&self, session_id: &str, task_id: &str) {
556 if let Ok(mut tasks) = self.inner.wait_registered_tasks.lock() {
557 tasks
558 .entry(session_id.to_string())
559 .or_default()
560 .insert(task_id.to_string());
561 }
562 }
563
564 pub fn begin_wait_mode_session(&self, session_id: &str, task_id: &str) {
565 if let Ok(mut active) = self.inner.active_wait_sessions.lock() {
566 *active.entry(session_id.to_string()).or_insert(0) += 1;
567 }
568 self.register_foreground_task(session_id, task_id);
569 if let Ok(mut detach) = self.inner.wait_detach_sessions.lock() {
570 detach.remove(session_id);
571 }
572 }
573
574 pub fn unregister_foreground_task(&self, session_id: &str, task_id: &str) {
575 if let Ok(mut tasks) = self.inner.wait_registered_tasks.lock() {
576 if let Some(session_tasks) = tasks.get_mut(session_id) {
577 session_tasks.remove(task_id);
578 if session_tasks.is_empty() {
579 tasks.remove(session_id);
580 }
581 }
582 }
583 }
584
585 pub fn end_wait_mode_session(&self, session_id: &str, task_id: &str) {
586 let no_active_wait = if let Ok(mut active) = self.inner.active_wait_sessions.lock() {
587 match active.get_mut(session_id) {
588 Some(count) if *count > 1 => *count -= 1,
589 Some(_) => {
590 active.remove(session_id);
591 }
592 None => {}
593 }
594 !active.contains_key(session_id)
595 } else {
596 false
597 };
598 self.unregister_foreground_task(session_id, task_id);
599 if no_active_wait {
600 if let Ok(mut detach) = self.inner.wait_detach_sessions.lock() {
601 detach.remove(session_id);
602 }
603 }
604 }
605
606 pub fn abort_inflight(&self, session_id: &str) -> Result<usize, String> {
610 let task_ids = self
611 .inner
612 .wait_registered_tasks
613 .lock()
614 .map(|mut tasks| tasks.remove(session_id).unwrap_or_default())
615 .map_err(|_| "wait registration lock poisoned".to_string())?;
616 if let Ok(mut active) = self.inner.active_wait_sessions.lock() {
617 active.remove(session_id);
618 }
619 if let Ok(mut detach) = self.inner.wait_detach_sessions.lock() {
620 detach.remove(session_id);
621 }
622
623 let mut killed = 0;
624 for task_id in task_ids {
625 let Some(task) = self.task_for_session(&task_id, session_id) else {
626 continue;
627 };
628 let is_terminal = task
629 .state
630 .lock()
631 .map(|state| state.metadata.status.is_terminal())
632 .map_err(|_| "background task lock poisoned".to_string())?;
633 if is_terminal {
634 continue;
635 }
636 let snapshot = self.kill_with_status_reason(
637 &task_id,
638 session_id,
639 BgTaskStatus::Killed,
640 Some("call_aborted".to_string()),
641 )?;
642 if snapshot.info.status == BgTaskStatus::Killed
643 && snapshot.info.status_reason.as_deref() == Some("call_aborted")
644 {
645 killed += 1;
646 }
647 }
648 Ok(killed)
649 }
650
651 pub fn signal_wait_mode_detach(&self, session_id: &str) -> bool {
652 let is_waiting = self
653 .inner
654 .active_wait_sessions
655 .lock()
656 .map(|active| active.get(session_id).copied().unwrap_or(0) > 0)
657 .unwrap_or(false);
658 if !is_waiting {
659 return false;
660 }
661 self.inner
662 .wait_detach_sessions
663 .lock()
664 .map(|mut detach| detach.insert(session_id.to_string()))
665 .unwrap_or(false)
666 }
667
668 pub fn active_wait_session_count(&self) -> usize {
671 self.inner
672 .active_wait_sessions
673 .lock()
674 .map(|active| active.len())
675 .unwrap_or(0)
676 }
677
678 pub fn take_wait_mode_detach(&self, session_id: &str) -> bool {
679 self.inner
680 .wait_detach_sessions
681 .lock()
682 .map(|mut detach| detach.remove(session_id))
683 .unwrap_or(false)
684 }
685
686 pub(crate) fn wait_mode_detach_pending(&self, session_id: &str) -> bool {
687 self.inner
688 .wait_detach_sessions
689 .lock()
690 .map(|detach| detach.contains(session_id))
691 .unwrap_or(false)
692 }
693
694 pub(crate) async fn terminal_transition_notified(&self) {
695 self.inner.terminal_transition.notified().await;
696 }
697
698 pub fn set_compressor<F>(&self, compressor: F)
703 where
704 F: Fn(&str, String) -> CompressionResult + Send + Sync + 'static,
705 {
706 self.set_compressor_with_exit_code(move |command, output, _exit_code| {
707 compressor(command, output)
708 });
709 }
710
711 pub fn set_compressor_with_exit_code<F>(&self, compressor: F)
712 where
713 F: Fn(&str, String, Option<i32>) -> CompressionResult + Send + Sync + 'static,
714 {
715 if let Ok(mut slot) = self.inner.compressor.lock() {
716 *slot = Some(Box::new(compressor));
717 }
718 }
719
720 pub(crate) fn compress_output(
723 &self,
724 command: &str,
725 output: String,
726 exit_code: Option<i32>,
727 ) -> CompressionResult {
728 let Ok(slot) = self.inner.compressor.lock() else {
729 return CompressionResult::new(output);
730 };
731 match slot.as_ref() {
732 Some(compressor) => compressor(command, output, exit_code),
733 None => CompressionResult::new(output),
734 }
735 }
736
737 fn ensure_terminal_output_cache(&self, task: &Arc<BgTask>) -> Option<TerminalOutputCache> {
738 let (metadata, buffer) = {
739 let state = task.state.lock().ok()?;
740 if !state.metadata.status.is_terminal() || state.metadata.mode == BgMode::Pty {
741 return None;
742 }
743 if let Some(cache) = state.terminal_output_cache.clone() {
744 return Some(cache);
745 }
746 (state.metadata.clone(), state.buffer.clone())
747 };
748
749 let mut cap_buffer = buffer.clone();
750 let disk_truncation = cap_buffer.enforce_terminal_cap();
751 let cache =
752 self.render_terminal_output(&metadata, &cap_buffer, disk_truncation, Some(&task.paths));
753 let mut state = task.state.lock().ok()?;
754 if !state.metadata.status.is_terminal() || state.metadata.mode == BgMode::Pty {
755 return None;
756 }
757 if let Some(existing) = state.terminal_output_cache.clone() {
758 return Some(existing);
759 }
760 state.terminal_output_cache = Some(cache.clone());
761 Some(cache)
762 }
763
764 fn render_terminal_output(
765 &self,
766 metadata: &PersistedTask,
767 buffer: &BgBuffer,
768 disk_truncation: DiskTruncation,
769 paths: Option<&TaskPaths>,
770 ) -> TerminalOutputCache {
771 let output_readable = buffer
772 .output_path()
773 .is_some_and(|path| self.is_session_owned_artifact_path(&metadata.session_id, &path));
774 let stderr_readable = buffer
775 .stderr_path()
776 .map(|path| self.is_session_owned_artifact_path(&metadata.session_id, path))
777 .unwrap_or(true);
778 let artifact_access = ArtifactRecoveryAccess {
779 task_id: metadata.task_id.clone(),
780 readable: output_readable && stderr_readable,
781 };
782
783 if metadata.mode == BgMode::Pty {
784 return TerminalOutputCache {
785 output_preview: String::new(),
786 output_truncated: false,
787 compression_input_line_count: None,
788 kind: TerminalOutputKind::Raw,
789 output_path: buffer.output_path().map(|path| path.display().to_string()),
790 stderr_path: buffer.stderr_path().map(|path| path.display().to_string()),
791 artifact_access,
792 recovery: None,
793 };
794 }
795
796 let mut rendered = if let Some(structured) = render_structured_output(
797 &metadata.command,
798 buffer,
799 disk_truncation,
800 artifact_access.clone(),
801 ) {
802 structured
803 } else if !metadata.compressed {
804 render_raw_passthrough(buffer, disk_truncation, artifact_access)
805 } else {
806 let raw = buffer.read_combined_head_tail(
807 COMPRESS_INPUT_CAP_BYTES,
808 COMPRESS_INPUT_HEAD_BYTES,
809 COMPRESS_INPUT_TAIL_BYTES,
810 );
811 let compressed = self.compress_output(&metadata.command, raw.text, metadata.exit_code);
812 render_compressed_with_recovery(
813 buffer,
814 compressed,
815 raw.truncated,
816 disk_truncation,
817 artifact_access,
818 )
819 };
820 normalize_piped_display_output(&mut rendered.output_preview);
821 append_pipeline_warning(&mut rendered, metadata, paths);
822 rendered
823 }
824
825 fn snapshot_with_terminal_cache(
826 &self,
827 task: &Arc<BgTask>,
828 preview_bytes: usize,
829 ) -> BgTaskSnapshot {
830 let mut snapshot = task.snapshot(preview_bytes);
831 self.maybe_compress_snapshot(task, &mut snapshot);
832 snapshot
833 }
834
835 fn post_terminal_transition(&self, task: &Arc<BgTask>, emit_frame: bool) -> Result<(), String> {
836 let should_sample = {
837 let mut state = task
838 .state
839 .lock()
840 .map_err(|_| "background task lock poisoned".to_string())?;
841 if !state.metadata.status.is_terminal() {
842 return Ok(());
843 }
844 if state.metadata.mode != BgMode::Pipes || state.descendant_sampling_started {
845 false
846 } else {
847 state.descendant_sampling_started = true;
848 true
849 }
850 };
851
852 let survivors_at_exit = should_sample
859 && self
860 .sample_task_process_group(task)
861 .is_some_and(|(members, omitted)| !members.is_empty() || omitted > 0);
862 if !survivors_at_exit {
863 let result = self.finish_terminal_transition(task, emit_frame);
864 self.inner.terminal_transition.notify_waiters();
865 return result;
866 }
867
868 let registry = self.clone();
872 let task = Arc::clone(task);
873 std::thread::Builder::new()
874 .name(format!("aft-bg-descendants-{}", task.task_id))
875 .spawn(move || {
876 std::thread::sleep(Duration::from_millis(300));
877 let settled = registry.sample_task_process_group(&task);
878 if let Err(error) = registry.finish_terminal_transition(&task, emit_frame) {
879 crate::slog_warn!(
880 "failed to finish background task {} after descendant sampling: {error}",
881 task.task_id
882 );
883 }
884 registry.inner.terminal_transition.notify_waiters();
885 if settled
886 .as_ref()
887 .is_some_and(|(members, omitted)| !members.is_empty() || *omitted > 0)
888 {
889 std::thread::sleep(Duration::from_millis(1_700));
890 let _ = registry.sample_task_process_group(&task);
891 }
892 })
893 .map_err(|error| format!("failed to start descendant sampler: {error}"))?;
894 Ok(())
895 }
896
897 fn sample_task_process_group(
898 &self,
899 task: &Arc<BgTask>,
900 ) -> Option<(Vec<LiveDescendant>, usize)> {
901 let pgid = task
902 .state
903 .lock()
904 .ok()
905 .and_then(|state| state.metadata.pgid)?;
906 let sample = live_process_group_members(pgid);
907 if let Ok(mut state) = task.state.lock() {
908 state.metadata.live_descendants = sample.as_ref().map(|(members, _)| members.clone());
909 state.metadata.live_descendants_omitted =
910 sample.as_ref().map(|(_, omitted)| *omitted).unwrap_or(0);
911 if let Err(error) = self.persist_task(&task.paths, &state.metadata) {
912 crate::slog_warn!(
913 "failed to persist live descendants for {}: {error}",
914 task.task_id
915 );
916 }
917 }
918 sample
919 }
920
921 fn finish_terminal_transition(
922 &self,
923 task: &Arc<BgTask>,
924 emit_frame: bool,
925 ) -> Result<(), String> {
926 let (metadata, buffer) = {
927 let state = task
928 .state
929 .lock()
930 .map_err(|_| "background task lock poisoned".to_string())?;
931 if !state.metadata.status.is_terminal() {
932 return Ok(());
933 }
934 (state.metadata.clone(), state.buffer.clone())
935 };
936
937 let cache = self.ensure_terminal_output_cache(task);
938 self.enqueue_completion_from_parts(
939 &metadata,
940 Some(&buffer),
941 None,
942 emit_frame,
943 cache.as_ref(),
944 );
945 Ok(())
946 }
947
948 fn persist_task(&self, paths: &TaskPaths, metadata: &PersistedTask) -> std::io::Result<()> {
949 let task = resolve_task_layout(&paths.session_dir, &paths.task_id)?;
950 write_task_at(&task, metadata)?;
951 self.dual_write_task(paths, metadata);
952 Ok(())
953 }
954
955 fn update_task_metadata<F>(
956 &self,
957 paths: &TaskPaths,
958 update: F,
959 ) -> std::io::Result<PersistedTask>
960 where
961 F: FnOnce(&mut PersistedTask),
962 {
963 let task = resolve_task_layout(&paths.session_dir, &paths.task_id)?;
964 let metadata = update_task_at(&task, update)?;
965 self.dual_write_task(paths, &metadata);
966 Ok(metadata)
967 }
968
969 fn dual_write_task(&self, paths: &TaskPaths, metadata: &PersistedTask) {
970 let pool = self.inner.db_pool.read().ok().and_then(|slot| slot.clone());
971 let Some(pool) = pool else {
972 return;
973 };
974 let harness = self
975 .inner
976 .db_harness
977 .read()
978 .ok()
979 .and_then(|slot| slot.clone());
980 let Some(harness) = harness else {
981 crate::slog_warn!(
982 "dual-write bash_task to DB skipped for {}: harness not configured",
983 metadata.task_id
984 );
985 return;
986 };
987 let row = match metadata.to_bash_task_row(&harness, paths) {
988 Ok(row) => row,
989 Err(error) => {
990 crate::slog_warn!(
991 "dual-write bash_task to DB failed for {}: {}",
992 metadata.task_id,
993 error
994 );
995 return;
996 }
997 };
998 let conn = match pool.lock() {
999 Ok(conn) => conn,
1000 Err(_) => {
1001 crate::slog_warn!(
1002 "dual-write bash_task to DB failed for {}: db mutex poisoned",
1003 metadata.task_id
1004 );
1005 return;
1006 }
1007 };
1008 if let Err(error) = crate::db::bash_tasks::upsert_bash_task(&conn, &row) {
1009 crate::slog_warn!(
1010 "dual-write bash_task to DB failed for {}: {}",
1011 metadata.task_id,
1012 error
1013 );
1014 }
1015 }
1016
1017 fn delete_gc_task_from_db(&self, metadata: &PersistedTask) {
1018 let pool = self.inner.db_pool.read().ok().and_then(|slot| slot.clone());
1019 let Some(pool) = pool else {
1020 return;
1021 };
1022 let harness = self
1023 .inner
1024 .db_harness
1025 .read()
1026 .ok()
1027 .and_then(|slot| slot.clone());
1028 let Some(harness) = harness else {
1029 crate::slog_warn!(
1030 "GC bash_task DB delete skipped for {}: harness not configured",
1031 metadata.task_id
1032 );
1033 return;
1034 };
1035 let conn = match pool.lock() {
1036 Ok(conn) => conn,
1037 Err(_) => {
1038 crate::slog_warn!(
1039 "GC bash_task DB delete failed for {}: db mutex poisoned",
1040 metadata.task_id
1041 );
1042 return;
1043 }
1044 };
1045 if let Err(error) = crate::db::bash_tasks::delete_delivered_terminal_bash_task(
1046 &conn,
1047 &harness,
1048 &metadata.session_id,
1049 &metadata.task_id,
1050 "persisted_gc_delivered_terminal",
1051 ) {
1052 crate::slog_warn!(
1053 "GC bash_task DB delete failed for {}: {}",
1054 metadata.task_id,
1055 error
1056 );
1057 }
1058 }
1059
1060 fn persisted_task_process_is_alive(metadata: &PersistedTask) -> bool {
1061 let child_pid = metadata.child_pid;
1062 let group_leader = metadata.pgid.and_then(|pid| u32::try_from(pid).ok());
1063 child_pid
1064 .into_iter()
1065 .chain(group_leader)
1066 .any(|pid| is_recorded_process_alive(pid, metadata.started_at))
1067 }
1068
1069 fn db_has_live_process_for_task(&self, task_id: &str) -> bool {
1070 let Some((harness, pool)) = self.db_harness_and_pool() else {
1071 return false;
1072 };
1073 let Ok(conn) = pool.lock() else {
1074 return false;
1075 };
1076 crate::db::bash_tasks::list_bash_tasks_by_id(&conn, &harness, task_id)
1077 .map(|rows| {
1078 rows.into_iter().any(|row| {
1079 let started_at = u64::try_from(row.started_at).unwrap_or_default();
1080 row.pid
1081 .and_then(|pid| u32::try_from(pid).ok())
1082 .into_iter()
1083 .chain(row.pgid.and_then(|pid| u32::try_from(pid).ok()))
1084 .any(|pid| is_recorded_process_alive(pid, started_at))
1085 })
1086 })
1087 .unwrap_or(false)
1088 }
1089
1090 fn db_harness_and_pool(&self) -> Option<(String, Arc<Mutex<TrackedConnection>>)> {
1091 let pool = self
1092 .inner
1093 .db_pool
1094 .read()
1095 .ok()
1096 .and_then(|slot| slot.clone())?;
1097 let harness = self
1098 .inner
1099 .db_harness
1100 .read()
1101 .ok()
1102 .and_then(|slot| slot.clone())?;
1103 Some((harness, pool))
1104 }
1105
1106 pub fn pending_pattern_matches_for_session(
1107 &self,
1108 session_id: &str,
1109 ) -> Vec<BashPatternMatchFrame> {
1110 let Some((harness, pool)) = self.db_harness_and_pool() else {
1111 return Vec::new();
1112 };
1113 let rows = {
1114 let Ok(conn) = pool.lock() else {
1115 return Vec::new();
1116 };
1117 match crate::db::bash_watches::list_bash_pattern_watches_for_session(
1118 &conn, &harness, session_id,
1119 ) {
1120 Ok(rows) => rows,
1121 Err(error) => {
1122 crate::slog_warn!(
1123 "failed to load pending bash watches for session {session_id}: {error}"
1124 );
1125 return Vec::new();
1126 }
1127 }
1128 };
1129
1130 rows.into_iter()
1131 .filter(|row| row.pending_match)
1132 .filter_map(|row| {
1133 let Some(match_text) = row.match_text else {
1134 crate::slog_warn!(
1135 "pending bash watch {}/{} has no match text",
1136 row.task_id,
1137 row.watch_id
1138 );
1139 return None;
1140 };
1141 let context = row.match_context.unwrap_or_else(|| match_text.clone());
1142 Some(if match_text == WATCH_TARGET_ERASED_TEXT {
1143 BashPatternMatchFrame::watch_target_erased(
1144 row.task_id,
1145 session_id,
1146 row.watch_id,
1147 match_text,
1148 context,
1149 )
1150 } else if match_text == WATCH_TASK_EXIT_TEXT {
1151 BashPatternMatchFrame::task_exit(row.task_id, session_id, match_text, context)
1152 } else {
1153 BashPatternMatchFrame::new(
1154 row.task_id,
1155 session_id,
1156 row.watch_id,
1157 match_text,
1158 row.match_offset.unwrap_or_default().max(0) as u64,
1159 context,
1160 row.once,
1161 )
1162 })
1163 })
1164 .collect()
1165 }
1166
1167 fn terminal_db_status_for_session(
1168 &self,
1169 session_id: &str,
1170 task_id: &str,
1171 storage_dir: &Path,
1172 ) -> Option<BgTaskSnapshot> {
1173 let (harness, pool) = self.db_harness_and_pool()?;
1174 let conn = pool.lock().ok()?;
1175 let row =
1176 crate::db::bash_tasks::get_bash_task(&conn, &harness, session_id, task_id).ok()??;
1177 if !task_bundle_is_absent(storage_dir, &row.session_id, &row.task_id) {
1178 return None;
1179 }
1180 let metadata = PersistedTask::from(row.clone());
1181 metadata
1182 .status
1183 .is_terminal()
1184 .then(|| terminal_db_row_snapshot(row, metadata))
1185 }
1186
1187 pub fn has_erased_watch_reference(&self, task_id: &str) -> bool {
1188 self.evaluate_erased_watch_targets();
1189 self.inner
1190 .watch_registry
1191 .lock()
1192 .map(|registry| registry.has_erased_task(task_id))
1193 .unwrap_or(false)
1194 }
1195
1196 pub(crate) fn evaluate_erased_watch_targets(&self) {
1197 let Some((harness, pool)) = self.db_harness_and_pool() else {
1198 return;
1199 };
1200 let watched_task_ids = self
1203 .inner
1204 .watch_registry
1205 .lock()
1206 .map(|registry| registry.watched_task_ids())
1207 .unwrap_or_default();
1208 if watched_task_ids.is_empty() {
1209 return;
1210 }
1211
1212 let candidates = watched_task_ids
1213 .into_iter()
1214 .filter_map(|task_id| {
1215 let task = self.task(&task_id)?;
1216 self.originating_session_has_live_route(&task.session_id)
1217 .then(|| (task.session_id.clone(), task_id))
1218 })
1219 .collect::<Vec<_>>();
1220 let erased_tasks = {
1221 let Ok(conn) = pool.lock() else {
1222 return;
1223 };
1224 let mut erased_tasks = Vec::new();
1225 for (session_id, task_id) in candidates {
1226 match crate::db::bash_tasks::get_bash_task(&conn, &harness, &session_id, &task_id) {
1227 Ok(None) => {
1228 if let Err(error) =
1229 crate::db::bash_watches::delete_bash_pattern_watches_for_task(
1230 &conn,
1231 &harness,
1232 &session_id,
1233 &task_id,
1234 )
1235 {
1236 crate::slog_warn!(
1237 "failed to retire durable watches for erased task {}: {error}",
1238 task_id
1239 );
1240 }
1241 erased_tasks.push((session_id, task_id));
1242 }
1243 Ok(Some(_)) => {}
1244 Err(error) => {
1245 crate::slog_warn!(
1246 "failed to inspect bash watch target {}: {error}",
1247 task_id
1248 );
1249 }
1250 }
1251 }
1252 erased_tasks
1253 };
1254
1255 for (session_id, task_id) in erased_tasks {
1256 let watch_ids = self
1257 .inner
1258 .watch_registry
1259 .lock()
1260 .map(|mut registry| registry.terminalize_erased_task(&task_id))
1261 .unwrap_or_default();
1262 for watch_id in watch_ids {
1263 self.emit_bash_watch_erased(&session_id, &task_id, &watch_id);
1264 }
1265 }
1266 }
1267
1268 fn persist_watch_registration(
1269 &self,
1270 session_id: &str,
1271 task_id: &str,
1272 watch_id: &str,
1273 pattern: &WatchPattern,
1274 once: bool,
1275 stdout_offset: u64,
1276 stderr_offset: u64,
1277 pty_offset: u64,
1278 ) {
1279 let Some((harness, pool)) = self.db_harness_and_pool() else {
1280 return;
1281 };
1282 let Ok(conn) = pool.lock() else {
1283 return;
1284 };
1285 let row = BashPatternWatchRow {
1286 harness,
1287 session_id: session_id.to_string(),
1288 task_id: task_id.to_string(),
1289 watch_id: watch_id.to_string(),
1290 pattern_kind: pattern.kind_name().to_string(),
1291 pattern: pattern.pattern_text().to_string(),
1292 once,
1293 created_at: unix_millis() as i64,
1294 stdout_offset: stdout_offset as i64,
1295 stderr_offset: stderr_offset as i64,
1296 pty_offset: pty_offset as i64,
1297 scanning: true,
1298 pending_match: false,
1299 match_text: None,
1300 match_offset: None,
1301 match_context: None,
1302 };
1303 if let Err(error) = crate::db::bash_watches::upsert_bash_pattern_watch(&conn, &row) {
1304 crate::slog_warn!(
1305 "persist bash_pattern_watch failed for {task_id}/{watch_id}: {error}"
1306 );
1307 }
1308 }
1309
1310 fn delete_persisted_watch(&self, session_id: &str, task_id: &str, watch_id: &str) {
1311 let Some((harness, pool)) = self.db_harness_and_pool() else {
1312 return;
1313 };
1314 let Ok(conn) = pool.lock() else {
1315 return;
1316 };
1317 if let Err(error) = crate::db::bash_watches::delete_bash_pattern_watch(
1318 &conn, &harness, session_id, task_id, watch_id,
1319 ) {
1320 crate::slog_warn!("delete bash_pattern_watch failed for {task_id}/{watch_id}: {error}");
1321 }
1322 }
1323
1324 fn delete_persisted_watches_for_task(&self, session_id: &str, task_id: &str) {
1325 let Some((harness, pool)) = self.db_harness_and_pool() else {
1326 return;
1327 };
1328 let Ok(conn) = pool.lock() else {
1329 return;
1330 };
1331 if let Err(error) = crate::db::bash_watches::delete_bash_pattern_watches_for_task(
1332 &conn, &harness, session_id, task_id,
1333 ) {
1334 crate::slog_warn!("delete bash_pattern_watches for {task_id} failed: {error}");
1335 }
1336 }
1337
1338 fn persist_watch_match(
1339 &self,
1340 session_id: &str,
1341 task_id: &str,
1342 pattern_match: &PatternMatch,
1343 stdout_offset: u64,
1344 stderr_offset: u64,
1345 pty_offset: u64,
1346 ) {
1347 let Some((harness, pool)) = self.db_harness_and_pool() else {
1348 return;
1349 };
1350 let Ok(conn) = pool.lock() else {
1351 return;
1352 };
1353 let Ok(Some(mut row)) = crate::db::bash_watches::get_bash_pattern_watch(
1354 &conn,
1355 &harness,
1356 session_id,
1357 task_id,
1358 &pattern_match.watch_id,
1359 ) else {
1360 return;
1361 };
1362 row.stdout_offset = stdout_offset as i64;
1363 row.stderr_offset = stderr_offset as i64;
1364 row.pty_offset = pty_offset as i64;
1365 row.pending_match = true;
1366 row.match_text = Some(pattern_match.match_text.clone());
1367 row.match_offset = Some(pattern_match.match_offset as i64);
1368 row.match_context = Some(pattern_match.context.clone());
1369 if pattern_match.once {
1370 row.scanning = false;
1373 }
1374 if let Err(error) = crate::db::bash_watches::upsert_bash_pattern_watch(&conn, &row) {
1375 crate::slog_warn!(
1376 "persist bash_pattern_watch match failed for {}/{}: {error}",
1377 task_id,
1378 pattern_match.watch_id
1379 );
1380 }
1381 }
1382
1383 fn persist_task_watch_cursors(
1384 &self,
1385 session_id: &str,
1386 task_id: &str,
1387 stdout_offset: u64,
1388 stderr_offset: u64,
1389 pty_offset: u64,
1390 ) {
1391 let Some((harness, pool)) = self.db_harness_and_pool() else {
1392 return;
1393 };
1394 let Ok(conn) = pool.lock() else {
1395 return;
1396 };
1397 if let Err(error) = crate::db::bash_watches::update_watch_offsets_for_task(
1398 &conn,
1399 &harness,
1400 session_id,
1401 task_id,
1402 stdout_offset as i64,
1403 stderr_offset as i64,
1404 pty_offset as i64,
1405 ) {
1406 crate::slog_warn!("persist bash_pattern_watch cursors failed for {task_id}: {error}");
1407 }
1408 }
1409
1410 fn watch_stream_cursors(&self, task_id: &str) -> (u64, u64, u64) {
1411 let Ok(registry) = self.inner.watch_registry.lock() else {
1412 return (0, 0, 0);
1413 };
1414 let stdout = registry
1415 .file_cursor(&format!("{task_id}:stdout"))
1416 .unwrap_or(0);
1417 let stderr = registry
1418 .file_cursor(&format!("{task_id}:stderr"))
1419 .unwrap_or(0);
1420 let pty = registry.file_cursor(&format!("{task_id}:pty")).unwrap_or(0);
1421 (stdout, stderr, pty)
1422 }
1423
1424 fn ack_persisted_watches_for_task(&self, session_id: &str, task_id: &str, task_terminal: bool) {
1427 let Some((harness, pool)) = self.db_harness_and_pool() else {
1428 return;
1429 };
1430 let Ok(conn) = pool.lock() else {
1431 return;
1432 };
1433 if task_terminal {
1434 let _ = crate::db::bash_watches::delete_bash_pattern_watches_for_task(
1435 &conn, &harness, session_id, task_id,
1436 );
1437 return;
1438 }
1439 let Ok(rows) = crate::db::bash_watches::list_bash_pattern_watches_for_task(
1440 &conn, &harness, session_id, task_id,
1441 ) else {
1442 return;
1443 };
1444 for mut row in rows {
1445 if row.once && (!row.scanning || row.pending_match) {
1446 let _ = crate::db::bash_watches::delete_bash_pattern_watch(
1447 &conn,
1448 &harness,
1449 session_id,
1450 task_id,
1451 &row.watch_id,
1452 );
1453 continue;
1454 }
1455 if row.pending_match {
1456 row.pending_match = false;
1457 row.match_text = None;
1458 row.match_offset = None;
1459 row.match_context = None;
1460 let _ = crate::db::bash_watches::upsert_bash_pattern_watch(&conn, &row);
1461 }
1462 }
1463 }
1464
1465 pub fn record_scanner_report(
1466 &self,
1467 task_id: &str,
1468 session_id: &str,
1469 scanner_report: Vec<PermissionAsk>,
1470 ) -> Result<(), String> {
1471 if scanner_report.is_empty() {
1472 return Ok(());
1473 }
1474 let task = self.task_for_session(task_id, session_id).ok_or_else(|| {
1475 "background task not found while recording scanner report".to_string()
1476 })?;
1477 let metadata = {
1478 let mut state = task
1479 .state
1480 .lock()
1481 .map_err(|_| "background task lock poisoned".to_string())?;
1482 state.metadata.scanner_report = scanner_report;
1483 state.metadata.clone()
1484 };
1485 self.persist_task(&task.paths, &metadata)
1486 .map_err(|error| format!("failed to persist scanner report: {error}"))
1487 }
1488
1489 pub fn configure_long_running_reminders(&self, enabled: bool, interval_ms: u64) {
1490 self.inner
1491 .long_running_reminder_enabled
1492 .store(enabled, Ordering::SeqCst);
1493 self.inner
1494 .long_running_reminder_interval_ms
1495 .store(interval_ms, Ordering::SeqCst);
1496 }
1497
1498 #[cfg(unix)]
1499 #[allow(clippy::too_many_arguments)]
1500 pub fn spawn(
1501 &self,
1502 spawn_plan: SpawnPlan,
1503 command: &str,
1504 session_id: String,
1505 workdir: PathBuf,
1506 env: HashMap<String, String>,
1507 timeout: Option<Duration>,
1508 storage_dir: PathBuf,
1509 max_running: usize,
1510 notify_on_completion: bool,
1511 compressed: bool,
1512 project_root: Option<PathBuf>,
1513 ) -> Result<String, String> {
1514 self.spawn_with_shell(
1515 spawn_plan,
1516 command,
1517 super::BashShell::Bash,
1518 resolve_posix_shell(),
1519 session_id,
1520 workdir,
1521 env,
1522 timeout,
1523 storage_dir,
1524 max_running,
1525 notify_on_completion,
1526 compressed,
1527 project_root,
1528 )
1529 }
1530
1531 #[cfg(unix)]
1532 #[allow(clippy::too_many_arguments)]
1533 pub fn spawn_with_shell(
1534 &self,
1535 spawn_plan: SpawnPlan,
1536 command: &str,
1537 shell: super::BashShell,
1538 shell_path: PathBuf,
1539 session_id: String,
1540 workdir: PathBuf,
1541 #[cfg_attr(not(target_os = "linux"), allow(unused_mut))] mut env: HashMap<String, String>,
1542 timeout: Option<Duration>,
1543 storage_dir: PathBuf,
1544 max_running: usize,
1545 notify_on_completion: bool,
1546 compressed: bool,
1547 project_root: Option<PathBuf>,
1548 ) -> Result<String, String> {
1549 self.start_watchdog();
1550
1551 #[cfg(target_os = "linux")]
1552 let linux_scope = env.remove(LINUX_SCOPE_ENV).as_deref() == Some("1");
1553 #[cfg(not(target_os = "linux"))]
1554 let linux_scope = false;
1555
1556 let running = self.running_count();
1557 if running >= max_running {
1558 #[cfg(unix)]
1559 if let Some(prepared) = spawn_plan.prepared_task() {
1560 let _ = delete_resolved_task(&prepared.resolved_task());
1561 }
1562 return Err(format!(
1563 "background bash task limit exceeded: {running} running (max {max_running})"
1564 ));
1565 }
1566
1567 let timeout = timeout.or(Some(DEFAULT_BG_TIMEOUT));
1568 let timeout_ms = timeout.map(|timeout| timeout.as_millis() as u64);
1569 let (spawn_plan, task_layout) = if let Some(prepared) = spawn_plan.prepared_task() {
1570 (spawn_plan.clone(), prepared.resolved_task())
1571 } else {
1572 let task = allocate_task_layout(&storage_dir, &session_id)
1573 .map_err(|error| format!("failed to create background task layout: {error}"))?;
1574 let root = project_root.as_deref().unwrap_or(&workdir);
1575 let environment =
1576 crate::sandbox_spawn::approved_payload_environment(&env, &std::env::temp_dir());
1577 let prepared = match crate::sandbox_spawn::prepare_task_payload(
1578 &task,
1579 command.as_bytes(),
1580 root,
1581 &workdir,
1582 &crate::sandbox_spawn::AuthenticatedPrincipal::FirstParty,
1583 &shell_path,
1584 &environment,
1585 ) {
1586 Ok(prepared) => prepared,
1587 Err(error) => {
1588 let _ = delete_resolved_task(&task);
1589 return Err(error);
1590 }
1591 };
1592 let task = prepared.resolved_task();
1593 (spawn_plan.with_prepared_task(prepared), task)
1594 };
1595 let task_id = task_layout.paths.task_id.clone();
1596 let paths = task_layout.paths.clone();
1597
1598 if self.task(&task_id).is_some() {
1599 let _ = delete_resolved_task(&task_layout);
1600 return Err("background task id collided with a live task".to_string());
1601 }
1602
1603 let mut metadata = PersistedTask::starting(
1604 task_id.clone(),
1605 session_id.clone(),
1606 command.to_string(),
1607 workdir.clone(),
1608 project_root,
1609 timeout_ms,
1610 notify_on_completion,
1611 compressed,
1612 );
1613 #[cfg(unix)]
1617 let capture_pipeline_status = {
1618 let pipeline = single_top_level_pipeline(command);
1619 let has_pipeline = pipeline.is_some();
1620 let capture = !shell.is_powershell()
1621 && should_capture_pipeline_status(&spawn_plan, has_pipeline, &shell_path);
1622 if !shell.is_powershell() {
1623 metadata.pipeline_segments = pipeline
1624 .as_ref()
1625 .map(|pipeline| {
1626 pipeline
1627 .segments
1628 .iter()
1629 .map(|segment| segment.label.clone())
1630 .collect()
1631 })
1632 .unwrap_or_default();
1633 if has_pipeline && !capture {
1634 metadata.pipeline_status_unavailable =
1635 Some(if spawn_plan.is_native_launcher() {
1636 "native sandbox launcher".to_string()
1637 } else {
1638 shell_path
1639 .file_name()
1640 .and_then(|name| name.to_str())
1641 .unwrap_or("unknown shell")
1642 .to_string()
1643 });
1644 }
1645 }
1646 capture
1647 };
1648 #[cfg(windows)]
1649 let capture_pipeline_status = false;
1650 attach_sandbox_metadata(&mut metadata, &spawn_plan);
1651 if let Err(error) = write_task_at(&task_layout, &metadata) {
1652 let _ = delete_resolved_task(&task_layout);
1653 return Err(format!(
1654 "failed to persist background task metadata: {error}"
1655 ));
1656 }
1657 self.dual_write_task(&paths, &metadata);
1658
1659 let mut io_handles =
1660 TaskIoHandles::create(&task_layout, BgMode::Pipes, capture_pipeline_status)
1661 .map_err(|error| format!("failed to pre-open task output handles: {error}"))?;
1662 let child = match spawn_detached_child(
1663 &spawn_plan,
1664 command,
1665 shell,
1666 &shell_path,
1667 &paths,
1668 &workdir,
1669 &env,
1670 &mut io_handles,
1671 capture_pipeline_status,
1672 linux_scope,
1673 ) {
1674 Ok(child) => child,
1675 Err(error) => {
1676 crate::slog_warn!("failed to spawn background bash task {task_id}; deleting partial bundle: {error}");
1677 let _ = delete_task_bundle(&paths);
1678 return Err(error);
1679 }
1680 };
1681
1682 let child_pid = child.id();
1683 metadata.mark_running(child_pid, child_pid as i32);
1684 self.persist_task(&paths, &metadata)
1685 .map_err(|e| format!("failed to persist running background task metadata: {e}"))?;
1686
1687 let task = Arc::new(BgTask {
1688 task_id: task_id.clone(),
1689 session_id,
1690 paths: paths.clone(),
1691 artifact_root: canonical_artifact_root(&paths),
1692 started: Instant::now(),
1693 last_reminder_at: Mutex::new(None),
1694 terminal_at: Mutex::new(None),
1695 state: Mutex::new(BgTaskState {
1696 metadata,
1697 runtime: TaskRuntime::Piped(Some(child)),
1698 io_handles: Some(io_handles),
1699 detached: false,
1700 child_exit_observed: false,
1701 descendant_sampling_started: false,
1702 buffer: BgBuffer::registered(&paths, BgMode::Pipes),
1703 terminal_output_cache: None,
1704 pending_terminal_override: None,
1705 }),
1706 });
1707
1708 self.record_live_delivery_session(&task.session_id);
1709 self.inner
1710 .tasks
1711 .lock()
1712 .map_err(|_| "background task registry lock poisoned".to_string())?
1713 .insert(task_id.clone(), task);
1714
1715 Ok(task_id)
1716 }
1717
1718 #[allow(clippy::too_many_arguments)]
1719 pub fn spawn_pty(
1720 &self,
1721 spawn_plan: SpawnPlan,
1722 command: &str,
1723 session_id: String,
1724 workdir: PathBuf,
1725 env: HashMap<String, String>,
1726 timeout: Option<Duration>,
1727 storage_dir: PathBuf,
1728 max_running: usize,
1729 notify_on_completion: bool,
1730 compressed: bool,
1731 project_root: Option<PathBuf>,
1732 rows: u16,
1733 cols: u16,
1734 ) -> Result<String, String> {
1735 self.spawn_pty_with_shell(
1736 spawn_plan,
1737 command,
1738 super::BashShell::Bash,
1739 super::resolve_shell_path(true, super::BashShell::Bash)
1740 .expect("POSIX shell must resolve for bash PTY"),
1741 session_id,
1742 workdir,
1743 env,
1744 timeout,
1745 storage_dir,
1746 max_running,
1747 notify_on_completion,
1748 compressed,
1749 project_root,
1750 rows,
1751 cols,
1752 )
1753 }
1754
1755 #[allow(clippy::too_many_arguments)]
1756 pub fn spawn_pty_with_shell(
1757 &self,
1758 spawn_plan: SpawnPlan,
1759 command: &str,
1760 shell: super::BashShell,
1761 shell_path: PathBuf,
1762 session_id: String,
1763 workdir: PathBuf,
1764 env: HashMap<String, String>,
1765 timeout: Option<Duration>,
1766 storage_dir: PathBuf,
1767 max_running: usize,
1768 notify_on_completion: bool,
1769 compressed: bool,
1770 project_root: Option<PathBuf>,
1771 rows: u16,
1772 cols: u16,
1773 ) -> Result<String, String> {
1774 self.start_watchdog();
1775
1776 let running = self.running_count();
1777 if running >= max_running {
1778 #[cfg(unix)]
1779 if let Some(prepared) = spawn_plan.prepared_task() {
1780 let _ = delete_resolved_task(&prepared.resolved_task());
1781 }
1782 return Err(format!(
1783 "background bash task limit exceeded: {running} running (max {max_running})"
1784 ));
1785 }
1786
1787 let timeout = timeout.or(Some(DEFAULT_BG_TIMEOUT));
1788 let timeout_ms = timeout.map(|timeout| timeout.as_millis() as u64);
1789 #[cfg(unix)]
1790 let (spawn_plan, task_layout) = if let Some(prepared) = spawn_plan.prepared_task() {
1791 (spawn_plan.clone(), prepared.resolved_task())
1792 } else {
1793 let task = allocate_task_layout(&storage_dir, &session_id)
1794 .map_err(|error| format!("failed to create PTY task layout: {error}"))?;
1795 let root = project_root.as_deref().unwrap_or(&workdir);
1796 let environment =
1797 crate::sandbox_spawn::approved_payload_environment(&env, &std::env::temp_dir());
1798 let prepared = match crate::sandbox_spawn::prepare_task_payload(
1799 &task,
1800 command.as_bytes(),
1801 root,
1802 &workdir,
1803 &crate::sandbox_spawn::AuthenticatedPrincipal::FirstParty,
1804 &shell_path,
1805 &environment,
1806 ) {
1807 Ok(prepared) => prepared,
1808 Err(error) => {
1809 let _ = delete_resolved_task(&task);
1810 return Err(error);
1811 }
1812 };
1813 let task = prepared.resolved_task();
1814 (spawn_plan.with_prepared_task(prepared), task)
1815 };
1816 #[cfg(windows)]
1817 let task_layout = allocate_task_layout(&storage_dir, &session_id)
1818 .map_err(|error| format!("failed to create PTY task layout: {error}"))?;
1819 let task_id = task_layout.paths.task_id.clone();
1820 let paths = task_layout.paths.clone();
1821
1822 let mut metadata = PersistedTask::starting(
1823 task_id.clone(),
1824 session_id.clone(),
1825 command.to_string(),
1826 workdir.clone(),
1827 project_root,
1828 timeout_ms,
1829 notify_on_completion,
1830 compressed,
1831 );
1832 attach_sandbox_metadata(&mut metadata, &spawn_plan);
1833 metadata.mode = BgMode::Pty;
1834 metadata.pty_rows = Some(rows);
1835 metadata.pty_cols = Some(cols);
1836 if let Err(error) = write_task_at(&task_layout, &metadata) {
1837 let _ = delete_resolved_task(&task_layout);
1838 return Err(format!(
1839 "failed to persist background task metadata: {error}"
1840 ));
1841 }
1842 self.dual_write_task(&paths, &metadata);
1843 let mut io_handles = TaskIoHandles::create(&task_layout, BgMode::Pty, false)
1844 .map_err(|error| format!("failed to pre-open PTY output handles: {error}"))?;
1845
1846 let runtime = match spawn_pty_for_command(
1847 &spawn_plan,
1848 &task_id,
1849 &session_id,
1850 command,
1851 shell,
1852 &shell_path,
1853 &paths,
1854 &workdir,
1855 &env,
1856 rows,
1857 cols,
1858 self.inner.wake_tx.clone(),
1859 &mut io_handles,
1860 ) {
1861 Ok(runtime) => runtime,
1862 Err(error) => {
1863 crate::slog_warn!(
1864 "failed to spawn PTY background bash task {task_id}; deleting partial bundle: {error}"
1865 );
1866 let _ = delete_task_bundle(&paths);
1867 return Err(error);
1868 }
1869 };
1870
1871 if let Some(child_pid) = runtime.child_pid {
1872 metadata.mark_running(child_pid, child_pid as i32);
1873 } else {
1874 metadata.status = BgTaskStatus::Running;
1875 metadata.pgid = None;
1876 }
1877 self.persist_task(&paths, &metadata)
1878 .map_err(|e| format!("failed to persist running background task metadata: {e}"))?;
1879
1880 let task = Arc::new(BgTask {
1881 task_id: task_id.clone(),
1882 session_id,
1883 paths: paths.clone(),
1884 artifact_root: canonical_artifact_root(&paths),
1885 started: Instant::now(),
1886 last_reminder_at: Mutex::new(None),
1887 terminal_at: Mutex::new(None),
1888 state: Mutex::new(BgTaskState {
1889 metadata,
1890 runtime: TaskRuntime::Pty(Some(runtime)),
1891 io_handles: Some(io_handles),
1892 detached: false,
1893 child_exit_observed: false,
1894 descendant_sampling_started: false,
1895 buffer: BgBuffer::registered(&paths, BgMode::Pty),
1896 terminal_output_cache: None,
1897 pending_terminal_override: None,
1898 }),
1899 });
1900
1901 self.record_live_delivery_session(&task.session_id);
1902 self.inner
1903 .tasks
1904 .lock()
1905 .map_err(|_| "background task registry lock poisoned".to_string())?
1906 .insert(task_id.clone(), task);
1907
1908 Ok(task_id)
1909 }
1910
1911 #[cfg(windows)]
1912 #[allow(clippy::too_many_arguments)]
1913 pub fn spawn(
1914 &self,
1915 spawn_plan: SpawnPlan,
1916 command: &str,
1917 session_id: String,
1918 workdir: PathBuf,
1919 env: HashMap<String, String>,
1920 timeout: Option<Duration>,
1921 storage_dir: PathBuf,
1922 max_running: usize,
1923 notify_on_completion: bool,
1924 compressed: bool,
1925 project_root: Option<PathBuf>,
1926 ) -> Result<String, String> {
1927 self.spawn_with_shell(
1928 spawn_plan,
1929 command,
1930 super::BashShell::Bash,
1931 PathBuf::from("cmd.exe"),
1932 session_id,
1933 workdir,
1934 env,
1935 timeout,
1936 storage_dir,
1937 max_running,
1938 notify_on_completion,
1939 compressed,
1940 project_root,
1941 )
1942 }
1943
1944 #[cfg(windows)]
1945 #[allow(clippy::too_many_arguments)]
1946 pub fn spawn_with_shell(
1947 &self,
1948 spawn_plan: SpawnPlan,
1949 command: &str,
1950 shell: super::BashShell,
1951 shell_path: PathBuf,
1952 session_id: String,
1953 workdir: PathBuf,
1954 env: HashMap<String, String>,
1955 timeout: Option<Duration>,
1956 storage_dir: PathBuf,
1957 max_running: usize,
1958 notify_on_completion: bool,
1959 compressed: bool,
1960 project_root: Option<PathBuf>,
1961 ) -> Result<String, String> {
1962 self.start_watchdog();
1963
1964 let running = self.running_count();
1965 if running >= max_running {
1966 #[cfg(unix)]
1967 if let Some(prepared) = spawn_plan.prepared_task() {
1968 let _ = delete_resolved_task(&prepared.resolved_task());
1969 }
1970 return Err(format!(
1971 "background bash task limit exceeded: {running} running (max {max_running})"
1972 ));
1973 }
1974
1975 let timeout = timeout.or(Some(DEFAULT_BG_TIMEOUT));
1976 let timeout_ms = timeout.map(|timeout| timeout.as_millis() as u64);
1977 let task_layout = allocate_task_layout(&storage_dir, &session_id)
1978 .map_err(|error| format!("failed to create background task layout: {error}"))?;
1979 let task_id = task_layout.paths.task_id.clone();
1980 let paths = task_layout.paths.clone();
1981
1982 let mut metadata = PersistedTask::starting(
1983 task_id.clone(),
1984 session_id.clone(),
1985 command.to_string(),
1986 workdir.clone(),
1987 project_root,
1988 timeout_ms,
1989 notify_on_completion,
1990 compressed,
1991 );
1992 attach_sandbox_metadata(&mut metadata, &spawn_plan);
1993 if let Err(error) = write_task_at(&task_layout, &metadata) {
1994 let _ = delete_resolved_task(&task_layout);
1995 return Err(format!(
1996 "failed to persist background task metadata: {error}"
1997 ));
1998 }
1999 self.dual_write_task(&paths, &metadata);
2000 let mut io_handles = TaskIoHandles::create(&task_layout, BgMode::Pipes, false)
2001 .map_err(|error| format!("failed to pre-open task output handles: {error}"))?;
2002
2003 let child = match spawn_detached_child(
2004 &spawn_plan,
2005 command,
2006 shell,
2007 &shell_path,
2008 &paths,
2009 &workdir,
2010 &env,
2011 &mut io_handles,
2012 false,
2013 false,
2014 ) {
2015 Ok(child) => child,
2016 Err(error) => {
2017 crate::slog_warn!("failed to spawn background bash task {task_id}; deleting partial bundle: {error}");
2018 let _ = delete_task_bundle(&paths);
2019 return Err(error);
2020 }
2021 };
2022
2023 let child_pid = child.id();
2024 metadata.status = BgTaskStatus::Running;
2025 metadata.child_pid = Some(child_pid);
2026 metadata.pgid = None;
2027 self.persist_task(&paths, &metadata)
2028 .map_err(|e| format!("failed to persist running background task metadata: {e}"))?;
2029
2030 let task = Arc::new(BgTask {
2031 task_id: task_id.clone(),
2032 session_id,
2033 paths: paths.clone(),
2034 artifact_root: canonical_artifact_root(&paths),
2035 started: Instant::now(),
2036 last_reminder_at: Mutex::new(None),
2037 terminal_at: Mutex::new(None),
2038 state: Mutex::new(BgTaskState {
2039 metadata,
2040 runtime: TaskRuntime::Piped(Some(child)),
2041 io_handles: Some(io_handles),
2042 detached: false,
2043 child_exit_observed: false,
2044 descendant_sampling_started: false,
2045 buffer: BgBuffer::registered(&paths, BgMode::Pipes),
2046 terminal_output_cache: None,
2047 pending_terminal_override: None,
2048 }),
2049 });
2050
2051 self.record_live_delivery_session(&task.session_id);
2052 self.inner
2053 .tasks
2054 .lock()
2055 .map_err(|_| "background task registry lock poisoned".to_string())?
2056 .insert(task_id.clone(), task);
2057
2058 Ok(task_id)
2059 }
2060
2061 pub fn write_pty(
2062 &self,
2063 task_id: &str,
2064 session_id: &str,
2065 input: &[u8],
2066 ) -> Result<usize, String> {
2067 let task = self
2068 .task_for_session(task_id, session_id)
2069 .ok_or_else(|| "task_not_found".to_string())?;
2070
2071 let writer = {
2072 let state = task
2073 .state
2074 .lock()
2075 .map_err(|_| "background task lock poisoned".to_string())?;
2076 if state.metadata.mode != BgMode::Pty {
2077 return Err("task_not_pty".to_string());
2078 }
2079 if state.metadata.status.is_terminal() {
2080 return Err("task_exited".to_string());
2081 }
2082 match &state.runtime {
2083 TaskRuntime::Pty(Some(runtime)) => Arc::clone(&runtime.writer),
2084 TaskRuntime::Pty(None) => return Err("task_exited".to_string()),
2085 TaskRuntime::Piped(_) => return Err("task_not_pty".to_string()),
2086 }
2087 };
2088
2089 let mut writer = writer
2090 .lock()
2091 .map_err(|_| "PTY writer lock poisoned".to_string())?;
2092 writer
2093 .write_all(input)
2094 .map_err(|error| format!("failed to write to PTY: {error}"))?;
2095 writer
2096 .flush()
2097 .map_err(|error| format!("failed to flush PTY writer: {error}"))?;
2098 Ok(input.len())
2099 }
2100
2101 pub fn replay_session(&self, storage_dir: &Path, session_id: &str) -> Result<(), String> {
2102 self.replay_session_inner(storage_dir, session_id, None)
2103 }
2104
2105 #[doc(hidden)]
2108 pub fn persisted_gc_thread(&self) -> Option<String> {
2109 self.inner
2110 .persisted_gc_thread
2111 .lock()
2112 .unwrap_or_else(std::sync::PoisonError::into_inner)
2113 .clone()
2114 }
2115
2116 pub fn replay_session_for_project(
2117 &self,
2118 storage_dir: &Path,
2119 session_id: &str,
2120 project_root: &Path,
2121 ) -> Result<(), String> {
2122 self.replay_session_inner(storage_dir, session_id, Some(project_root))
2123 }
2124
2125 fn retire_orphaned_watch_tombstones(&self, binding_session_id: &str) -> Result<(), String> {
2126 let Some((harness, pool)) = self.db_harness_and_pool() else {
2127 return Ok(());
2128 };
2129 let retired = {
2130 let conn = pool
2131 .lock()
2132 .map_err(|_| "background task database lock poisoned".to_string())?;
2133 let rows = crate::db::bash_watches::list_bash_pattern_watches(&conn, &harness)
2134 .map_err(|error| format!("failed to inspect persisted bash watches: {error}"))?;
2135 let mut retired = Vec::new();
2136 for row in rows {
2137 let task_exists = match crate::db::bash_tasks::get_bash_task(
2138 &conn,
2139 &harness,
2140 &row.session_id,
2141 &row.task_id,
2142 ) {
2143 Ok(task) => task.is_some(),
2144 Err(_) => true,
2149 };
2150 let is_erased_target = !task_exists
2151 || (row.pending_match
2152 && row.match_text.as_deref() == Some(WATCH_TARGET_ERASED_TEXT));
2153 if !is_erased_target
2154 || !self.should_retire_foreign_delivery(&row.session_id, binding_session_id)
2155 {
2156 continue;
2157 }
2158 let deleted = crate::db::bash_watches::delete_bash_pattern_watch(
2159 &conn,
2160 &harness,
2161 &row.session_id,
2162 &row.task_id,
2163 &row.watch_id,
2164 )
2165 .map_err(|error| {
2166 format!(
2167 "failed to retire orphaned bash watch {}/{}: {error}",
2168 row.task_id, row.watch_id
2169 )
2170 })?;
2171 if deleted > 0 {
2172 retired.push((row.session_id, row.task_id));
2173 }
2174 }
2175 retired
2176 };
2177
2178 for (originating_session_id, task_id) in retired {
2179 self.clear_task_watch_state(&task_id);
2180 if let Ok(mut registry) = self.inner.watch_registry.lock() {
2181 registry.forget_erased_task(&task_id);
2182 }
2183 crate::slog_info!(
2184 "retired orphaned background watch tombstone: task_id={} originating_session={}",
2185 task_id,
2186 originating_session_id
2187 );
2188 }
2189 Ok(())
2190 }
2191
2192 fn retire_already_reaped_orphaned_completion(
2193 &self,
2194 originating_session_id: &str,
2195 task_id: &str,
2196 ) {
2197 if let Some((harness, pool)) = self.db_harness_and_pool() {
2198 match pool.lock() {
2199 Ok(conn) => {
2200 if let Err(error) = crate::db::bash_tasks::delete_bash_task(
2201 &conn,
2202 &harness,
2203 originating_session_id,
2204 task_id,
2205 ) {
2206 crate::slog_warn!(
2207 "failed to delete already-reaped orphaned background completion row: task_id={task_id} error={error}"
2208 );
2209 }
2210 }
2211 Err(_) => crate::slog_warn!(
2212 "failed to delete already-reaped orphaned background completion row: task_id={task_id} error=database_lock_poisoned"
2213 ),
2214 }
2215 }
2216 let _ = self.remove_pending_completion(task_id);
2217 self.ack_persisted_watches_for_task(originating_session_id, task_id, true);
2218 crate::slog_warn!(
2219 "orphaned completion already reaped: task_id={task_id} reason=layout_missing"
2220 );
2221 }
2222
2223 fn retire_rehydrated_orphaned_completion(
2224 &self,
2225 task: &Arc<BgTask>,
2226 binding_session_id: &str,
2227 ) -> Result<(), String> {
2228 let should_retire = task
2229 .state
2230 .lock()
2231 .map_err(|_| "background task lock poisoned".to_string())
2232 .map(|state| {
2233 state.metadata.status.is_terminal()
2234 && !state.metadata.completion_delivered
2235 && (self.should_retire_foreign_delivery(&task.session_id, binding_session_id)
2236 || !task.paths.dir.exists())
2237 })?;
2238 if !should_retire {
2239 return Ok(());
2240 }
2241
2242 match task.set_completion_delivered(true, self) {
2243 Ok(()) => {
2244 let _ = self.remove_pending_completion(&task.task_id);
2245 self.ack_persisted_watches_for_task(&task.session_id, &task.task_id, true);
2246 crate::slog_info!(
2247 "retired orphaned background completion: task_id={} originating_session={}",
2248 task.task_id,
2249 task.session_id
2250 );
2251 }
2252 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
2253 if let Ok(mut state) = task.state.lock() {
2254 state.metadata.completion_delivered = true;
2255 }
2256 self.retire_already_reaped_orphaned_completion(&task.session_id, &task.task_id);
2257 }
2258 Err(error) => {
2259 return Err(format!("failed to update completion delivery: {error}"));
2260 }
2261 }
2262 Ok(())
2263 }
2264
2265 fn enqueue_replay_completion_if_needed(
2266 &self,
2267 metadata: &mut PersistedTask,
2268 paths: &TaskPaths,
2269 binding_session_id: &str,
2270 ) -> Result<(), String> {
2271 if metadata.status.is_terminal()
2272 && !metadata.completion_delivered
2273 && self.should_retire_foreign_delivery(&metadata.session_id, binding_session_id)
2274 {
2275 metadata.completion_delivered = true;
2276 self.persist_task(paths, metadata).map_err(|error| {
2277 format!(
2278 "failed to retire orphaned background completion {}: {error}",
2279 metadata.task_id
2280 )
2281 })?;
2282 let _ = self.remove_pending_completion(&metadata.task_id);
2283 self.ack_persisted_watches_for_task(&metadata.session_id, &metadata.task_id, true);
2284 crate::slog_info!(
2285 "retired orphaned background completion: task_id={} originating_session={}",
2286 metadata.task_id,
2287 metadata.session_id
2288 );
2289 return Ok(());
2290 }
2291
2292 self.enqueue_completion_if_needed(metadata, Some(paths), false);
2293 Ok(())
2294 }
2295
2296 fn replay_session_inner(
2297 &self,
2298 storage_dir: &Path,
2299 session_id: &str,
2300 project_root: Option<&Path>,
2301 ) -> Result<(), String> {
2302 self.record_live_delivery_session(session_id);
2306 self.retire_orphaned_watch_tombstones(session_id)?;
2307 self.start_watchdog();
2308 if !self.inner.persisted_gc_started.swap(true, Ordering::SeqCst) {
2309 let registry = self.clone();
2319 let storage_dir = storage_dir.to_path_buf();
2320 let spawned = std::thread::Builder::new()
2321 .name("aft-bash-task-gc".to_string())
2322 .spawn(move || {
2323 if let Err(error) = registry.maybe_gc_persisted(&storage_dir) {
2324 crate::slog_warn!("failed to GC persisted background bash tasks: {error}");
2325 }
2326 });
2327 if let Err(error) = spawned {
2328 crate::slog_warn!("failed to spawn persisted background task GC: {error}");
2329 }
2330 }
2331
2332 let canonical_project = project_root.map(canonicalized_path);
2333 let tasks = match self.replay_session_from_db(session_id, project_root) {
2345 Some(Ok(tasks)) if !tasks.is_empty() => tasks,
2346 Some(Ok(_)) => {
2347 let disk_tasks = self.replay_session_from_disk(storage_dir, session_id)?;
2348 if !disk_tasks.is_empty() {
2349 crate::slog_info!(
2350 "bash task replay: 0 in DB for session {}, {} from disk fallback",
2351 session_id,
2352 disk_tasks.len()
2353 );
2354 }
2355 disk_tasks
2356 }
2357 Some(Err(error)) => {
2358 crate::slog_warn!(
2359 "bash task replay DB lookup failed for session {}; falling back to disk: {}",
2360 session_id,
2361 error
2362 );
2363 self.replay_session_from_disk(storage_dir, session_id)?
2364 }
2365 None => {
2366 self.replay_session_from_disk(storage_dir, session_id)?
2368 }
2369 };
2370
2371 for mut metadata in tasks {
2372 if project_root.is_none() && metadata.session_id != session_id {
2373 continue;
2374 }
2375 if let Some(canonical_project) = canonical_project.as_deref() {
2376 let metadata_project = metadata.project_root.as_deref().map(canonicalized_path);
2377 if metadata_project.as_deref() != Some(canonical_project) {
2378 continue;
2379 }
2380 }
2381
2382 if validate_task_id(&metadata.task_id).is_err() {
2383 crate::slog_warn!(
2384 "ignoring persisted background task with invalid id {:?}",
2385 metadata.task_id
2386 );
2387 continue;
2388 }
2389 if let Some(task) = self.task(&metadata.task_id) {
2393 self.retire_rehydrated_orphaned_completion(&task, session_id)?;
2394 continue;
2395 }
2396 let session_dir = session_tasks_dir(storage_dir, &metadata.session_id);
2397 let resolved = match resolve_task_layout(&session_dir, &metadata.task_id) {
2398 Ok(task) => task,
2399 Err(error) => {
2400 if error.kind() == std::io::ErrorKind::NotFound
2401 && !Self::persisted_task_process_is_alive(&metadata)
2402 && (self.should_retire_foreign_delivery(&metadata.session_id, session_id)
2403 || &metadata.session_id != session_id)
2404 {
2405 self.retire_already_reaped_orphaned_completion(
2406 &metadata.session_id,
2407 &metadata.task_id,
2408 );
2409 continue;
2410 }
2411 if Self::persisted_task_process_is_alive(&metadata) {
2412 crate::slog_warn!(
2413 "refusing to quarantine unresolved live background task {}: {error}",
2414 metadata.task_id
2415 );
2416 continue;
2417 }
2418 crate::slog_warn!(
2419 "quarantining unresolved background task {}: {error}",
2420 metadata.task_id
2421 );
2422 let _ = quarantine_task_layout(
2423 storage_dir,
2424 &session_dir,
2425 &metadata.task_id,
2426 "invalid",
2427 );
2428 continue;
2429 }
2430 };
2431 match read_task_at(&resolved) {
2432 Ok(disk)
2433 if disk.task_id == metadata.task_id
2434 && disk.session_id == metadata.session_id => {}
2435 Ok(_) | Err(_) => {
2436 if Self::persisted_task_process_is_alive(&metadata) {
2437 crate::slog_warn!(
2438 "refusing to quarantine mismatched live background task {}",
2439 metadata.task_id
2440 );
2441 continue;
2442 }
2443 let _ = quarantine_task_layout(
2444 storage_dir,
2445 &session_dir,
2446 &metadata.task_id,
2447 "mismatch",
2448 );
2449 continue;
2450 }
2451 }
2452 let paths = resolved.paths;
2453 match metadata.status {
2454 BgTaskStatus::Starting => {
2455 let completion_was_delivered = metadata.completion_delivered;
2456 metadata.mark_terminal(
2457 BgTaskStatus::Failed,
2458 None,
2459 Some("spawn aborted".to_string()),
2460 );
2461 metadata.completion_delivered |= completion_was_delivered;
2462 let _ = self.persist_task(&paths, &metadata);
2463 self.enqueue_replay_completion_if_needed(&mut metadata, &paths, session_id)?;
2464 self.insert_rehydrated_task(metadata, paths, true)?;
2465 }
2466 BgTaskStatus::Running | BgTaskStatus::Killing => {
2467 if metadata.mode == BgMode::Pty {
2468 if let Ok(Some(marker)) = read_exit_marker(&paths) {
2469 let completion_was_delivered = metadata.completion_delivered;
2470 metadata = terminal_metadata_from_marker(metadata, marker, None);
2471 metadata.completion_delivered |= completion_was_delivered;
2472 let _ = self.persist_task(&paths, &metadata);
2473 self.enqueue_replay_completion_if_needed(
2474 &mut metadata,
2475 &paths,
2476 session_id,
2477 )?;
2478 self.insert_rehydrated_task(metadata, paths, true)?;
2479 } else if metadata.status.is_terminal() {
2480 self.insert_rehydrated_task(metadata, paths, true)?;
2481 } else {
2482 let completion_was_delivered = metadata.completion_delivered;
2483 metadata.mark_terminal(
2484 BgTaskStatus::Killed,
2485 None,
2486 Some("pty_lost_on_bridge_restart".to_string()),
2487 );
2488 metadata.completion_delivered |= completion_was_delivered;
2489 let _ = self.persist_task(&paths, &metadata);
2490 self.enqueue_replay_completion_if_needed(
2491 &mut metadata,
2492 &paths,
2493 session_id,
2494 )?;
2495 self.insert_rehydrated_task(metadata, paths, true)?;
2496 }
2497 } else if let Ok(Some(marker)) = read_exit_marker(&paths) {
2498 let reason = (metadata.status == BgTaskStatus::Killing).then(|| {
2499 "recovered from inconsistent killing state on replay".to_string()
2500 });
2501 if reason.is_some() {
2502 crate::slog_warn!("background task {} had killing state with exit marker; preferring marker",
2503 metadata.task_id);
2504 }
2505 let completion_was_delivered = metadata.completion_delivered;
2506 metadata = terminal_metadata_from_marker(metadata, marker, reason);
2507 metadata.completion_delivered |= completion_was_delivered;
2508 let _ = self.persist_task(&paths, &metadata);
2509 self.enqueue_replay_completion_if_needed(
2510 &mut metadata,
2511 &paths,
2512 session_id,
2513 )?;
2514 self.insert_rehydrated_task(metadata, paths, true)?;
2515 } else if metadata.status == BgTaskStatus::Killing {
2516 let _ = write_kill_marker_if_absent(&paths);
2517 let completion_was_delivered = metadata.completion_delivered;
2518 metadata.mark_terminal(
2519 BgTaskStatus::Killed,
2520 None,
2521 Some("recovered from inconsistent killing state on replay".to_string()),
2522 );
2523 metadata.completion_delivered |= completion_was_delivered;
2524 let _ = self.persist_task(&paths, &metadata);
2525 self.enqueue_replay_completion_if_needed(
2526 &mut metadata,
2527 &paths,
2528 session_id,
2529 )?;
2530 self.insert_rehydrated_task(metadata, paths, true)?;
2531 } else if Self::persisted_task_process_is_alive(&metadata) {
2532 self.insert_rehydrated_task(metadata, paths, true)?;
2533 } else {
2534 let completion_was_delivered = metadata.completion_delivered;
2535 metadata.mark_terminal(
2536 BgTaskStatus::FateUnknown,
2537 None,
2538 Some(restart_fate_unknown_reason(&metadata, &paths)),
2539 );
2540 metadata.completion_delivered |= completion_was_delivered;
2541 let _ = self.persist_task(&paths, &metadata);
2542 self.enqueue_replay_completion_if_needed(
2543 &mut metadata,
2544 &paths,
2545 session_id,
2546 )?;
2547 self.insert_rehydrated_task(metadata, paths, true)?;
2548 }
2549 }
2550 _ if metadata.status.is_terminal() => {
2551 self.enqueue_replay_completion_if_needed(&mut metadata, &paths, session_id)?;
2557 self.insert_rehydrated_task(metadata, paths, true)?;
2558 }
2559 _ => {}
2560 }
2561 }
2562
2563 Ok(())
2564 }
2565
2566 fn replay_session_from_db(
2567 &self,
2568 session_id: &str,
2569 project_root: Option<&Path>,
2570 ) -> Option<Result<Vec<PersistedTask>, String>> {
2571 let pool = self
2572 .inner
2573 .db_pool
2574 .read()
2575 .ok()
2576 .and_then(|slot| slot.clone())?;
2577 let harness = self
2578 .inner
2579 .db_harness
2580 .read()
2581 .ok()
2582 .and_then(|slot| slot.clone())?;
2583 let conn = match pool.lock() {
2584 Ok(conn) => conn,
2585 Err(_) => return Some(Err("db mutex poisoned".to_string())),
2586 };
2587 let rows = if let Some(project_root) = project_root {
2588 let project_key = crate::path_identity::project_scope_key(project_root);
2589 crate::db::bash_tasks::list_replayable_bash_tasks_for_project(
2590 &conn,
2591 &harness,
2592 &project_key,
2593 )
2594 } else {
2595 crate::db::bash_tasks::list_bash_tasks_for_session(&conn, &harness, session_id)
2596 };
2597 Some(
2598 rows.map(|rows| rows.into_iter().map(PersistedTask::from).collect())
2599 .map_err(|error| error.to_string()),
2600 )
2601 }
2602
2603 fn replay_session_from_disk(
2604 &self,
2605 storage_dir: &Path,
2606 session_id: &str,
2607 ) -> Result<Vec<PersistedTask>, String> {
2608 let dir = session_tasks_dir(storage_dir, session_id);
2609 if !dir.exists() {
2610 return Ok(Vec::new());
2611 }
2612
2613 let (task_ids, invalid_entries) = discover_task_ids(&dir)
2614 .map_err(|error| format!("failed to discover background task layouts: {error}"))?;
2615 for entry in invalid_entries {
2616 if let Err(error) = quarantine_invalid_entry(storage_dir, &dir, &entry) {
2617 crate::slog_warn!(
2618 "failed to quarantine invalid background task entry {:?}: {error}",
2619 entry
2620 );
2621 }
2622 }
2623
2624 let mut tasks = Vec::new();
2625 for task_id in task_ids {
2626 let task = match resolve_task_layout(&dir, &task_id) {
2627 Ok(task) => task,
2628 Err(error)
2629 if error.kind() == std::io::ErrorKind::NotFound
2630 && uninitialized_layout_is_recent(
2631 &dir,
2632 &task_id,
2633 Duration::from_secs(5 * 60),
2634 )
2635 .unwrap_or(false) =>
2636 {
2637 continue;
2638 }
2639 Err(error) => {
2640 if self.db_has_live_process_for_task(&task_id) {
2641 crate::slog_warn!(
2642 "refusing to quarantine unresolved live background task {task_id} during replay: {error}"
2643 );
2644 continue;
2645 }
2646 crate::slog_warn!(
2647 "quarantining unresolved background task {task_id} during replay: {error}"
2648 );
2649 let _ = quarantine_task_layout(storage_dir, &dir, &task_id, "invalid");
2650 continue;
2651 }
2652 };
2653 match read_task_at(&task) {
2654 Ok(metadata) if metadata.session_id == session_id => tasks.push(metadata),
2655 Ok(_) => {
2656 crate::slog_warn!(
2657 "quarantining background task {task_id} with mismatched session metadata"
2658 );
2659 let _ = quarantine_task_layout(storage_dir, &dir, &task_id, "mismatch");
2660 }
2661 Err(error) => {
2662 if self.db_has_live_process_for_task(&task_id) {
2663 crate::slog_warn!(
2664 "refusing to quarantine unreadable live background task {task_id} during replay: {error}"
2665 );
2666 continue;
2667 }
2668 crate::slog_warn!(
2669 "quarantining invalid background task metadata {task_id} during replay: {error}"
2670 );
2671 let _ = quarantine_task_layout(storage_dir, &dir, &task_id, "invalid");
2672 }
2673 }
2674 }
2675 Ok(tasks)
2676 }
2677
2678 pub fn register_watch(
2679 &self,
2680 task_id: String,
2681 pattern: WatchPattern,
2682 once: bool,
2683 ) -> Result<String, &'static str> {
2684 let task = self.task(&task_id).ok_or("task_not_found")?;
2685 self.record_live_delivery_session(&task.session_id);
2686 validate_task_id(&task_id).map_err(|_| "invalid_task_id")?;
2687 let (mode, terminal_at_registration) = task
2688 .state
2689 .lock()
2690 .map(|state| {
2691 (
2692 state.metadata.mode.clone(),
2693 state.metadata.status.is_terminal(),
2694 )
2695 })
2696 .map_err(|_| "background_task_lock_poisoned")?;
2697 let mut stdout = (mode == BgMode::Pipes)
2698 .then(|| open_task_artifact(&task.paths, TaskArtifact::Stdout))
2699 .transpose()
2700 .map_err(|_| "artifact_refused")?;
2701 let mut stderr = (mode == BgMode::Pipes)
2702 .then(|| open_task_artifact(&task.paths, TaskArtifact::Stderr))
2703 .transpose()
2704 .map_err(|_| "artifact_refused")?;
2705 let mut pty = (mode == BgMode::Pty)
2706 .then(|| open_task_artifact(&task.paths, TaskArtifact::Pty))
2707 .transpose()
2708 .map_err(|_| "artifact_refused")?;
2709
2710 let mut terminal_matches = Vec::new();
2711 let scanned_terminal = terminal_at_registration;
2712 let watch_id = {
2713 let mut registry = self
2714 .inner
2715 .watch_registry
2716 .lock()
2717 .map_err(|_| "watch_registry_poisoned")?;
2718 let watch_id = registry.register(task_id.clone(), pattern.clone(), once)?;
2719 match &mode {
2720 BgMode::Pipes => {
2721 let stdout_key = format!("{task_id}:stdout");
2722 let stderr_key = format!("{task_id}:stderr");
2723 if terminal_at_registration {
2724 registry.set_file_cursor(&stdout_key, 0);
2725 registry.set_file_cursor(&stderr_key, 0);
2726 terminal_matches.extend(registry.scan_file_new_bytes(
2727 &stdout_key,
2728 &task_id,
2729 stdout.as_mut().expect("pipe stdout opened"),
2730 ));
2731 terminal_matches.extend(registry.scan_file_new_bytes(
2732 &stderr_key,
2733 &task_id,
2734 stderr.as_mut().expect("pipe stderr opened"),
2735 ));
2736 } else {
2737 registry.prime_file_cursor(
2738 &stdout_key,
2739 stdout.as_ref().expect("pipe stdout opened"),
2740 );
2741 registry.prime_file_cursor(
2742 &stderr_key,
2743 stderr.as_ref().expect("pipe stderr opened"),
2744 );
2745 }
2746 }
2747 BgMode::Pty => {
2748 let pty_key = format!("{task_id}:pty");
2749 if terminal_at_registration {
2750 registry.set_file_cursor(&pty_key, 0);
2751 terminal_matches.extend(registry.scan_file_new_bytes(
2752 &pty_key,
2753 &task_id,
2754 pty.as_mut().expect("PTY artifact opened"),
2755 ));
2756 } else {
2757 registry.prime_file_cursor(
2758 &pty_key,
2759 pty.as_ref().expect("PTY artifact opened"),
2760 );
2761 }
2762 }
2763 }
2764 watch_id
2765 };
2766
2767 let (stdout_offset, stderr_offset, pty_offset) = self.watch_stream_cursors(&task_id);
2768 self.persist_watch_registration(
2769 &task.session_id,
2770 &task_id,
2771 &watch_id,
2772 &pattern,
2773 once,
2774 stdout_offset,
2775 stderr_offset,
2776 pty_offset,
2777 );
2778
2779 if task.is_terminal() {
2780 if !scanned_terminal {
2781 terminal_matches = {
2782 let mut registry = self
2783 .inner
2784 .watch_registry
2785 .lock()
2786 .map_err(|_| "watch_registry_poisoned")?;
2787 match &mode {
2788 BgMode::Pipes => {
2789 let stdout_key = format!("{task_id}:stdout");
2790 let stderr_key = format!("{task_id}:stderr");
2791 registry.set_file_cursor(&stdout_key, 0);
2792 registry.set_file_cursor(&stderr_key, 0);
2793 let mut matches = registry.scan_file_new_bytes(
2794 &stdout_key,
2795 &task_id,
2796 stdout.as_mut().expect("pipe stdout opened"),
2797 );
2798 matches.extend(registry.scan_file_new_bytes(
2799 &stderr_key,
2800 &task_id,
2801 stderr.as_mut().expect("pipe stderr opened"),
2802 ));
2803 matches
2804 }
2805 BgMode::Pty => {
2806 let pty_key = format!("{task_id}:pty");
2807 registry.set_file_cursor(&pty_key, 0);
2808 registry.scan_file_new_bytes(
2809 &pty_key,
2810 &task_id,
2811 pty.as_mut().expect("PTY artifact opened"),
2812 )
2813 }
2814 }
2815 };
2816 }
2817
2818 let (stdout_offset, stderr_offset, pty_offset) = self.watch_stream_cursors(&task_id);
2819 let (watch_controlled, watch_matched) = self.task_watch_state(&task_id);
2820 if terminal_matches.is_empty() && (!watch_controlled || watch_matched) {
2821 if watch_matched {
2822 self.clear_task_watch_state(&task_id);
2823 }
2824 return Ok(watch_id);
2825 }
2826
2827 let completion = self
2828 .remove_pending_completion(&task_id)
2829 .or_else(|| self.completion_snapshot_for_task(&task));
2830 if terminal_matches.is_empty() {
2831 if let Some(completion) = completion.as_ref() {
2832 self.record_bash_watch_exit(completion, true);
2833 }
2834 } else {
2835 for pattern_match in &terminal_matches {
2836 self.persist_watch_match(
2837 &task.session_id,
2838 &task_id,
2839 pattern_match,
2840 stdout_offset,
2841 stderr_offset,
2842 pty_offset,
2843 );
2844 self.emit_bash_pattern_match(&task.session_id, pattern_match.clone());
2845 }
2846 }
2847 self.clear_task_watch_state(&task_id);
2848 }
2849
2850 Ok(watch_id)
2851 }
2852
2853 pub fn unregister_watch(&self, task_id: &str, watch_id: &str) {
2854 let session_id = self.task(task_id).map(|task| task.session_id.clone());
2855 if let Ok(mut registry) = self.inner.watch_registry.lock() {
2856 registry.unregister(task_id, watch_id);
2857 }
2858 if let Some(session_id) = session_id {
2859 self.delete_persisted_watch(&session_id, task_id, watch_id);
2860 }
2861 }
2862
2863 pub fn active_watch_count(&self, task_id: &str) -> usize {
2864 self.inner
2865 .watch_registry
2866 .lock()
2867 .map(|registry| registry.active_count(task_id))
2868 .unwrap_or(0)
2869 }
2870
2871 fn task_watch_state(&self, task_id: &str) -> (bool, bool) {
2872 self.inner
2873 .watch_registry
2874 .lock()
2875 .map(|registry| {
2876 (
2877 registry.has_controlled_task(task_id),
2878 registry.has_matched_task(task_id),
2879 )
2880 })
2881 .unwrap_or((false, false))
2882 }
2883
2884 fn task_has_watch_control(&self, task_id: &str) -> bool {
2885 self.inner
2886 .watch_registry
2887 .lock()
2888 .map(|registry| registry.has_controlled_task(task_id))
2889 .unwrap_or(false)
2890 }
2891
2892 fn clear_task_watch_state(&self, task_id: &str) {
2893 if let Ok(mut registry) = self.inner.watch_registry.lock() {
2894 registry.clear_task(task_id);
2895 }
2896 }
2897
2898 pub(crate) fn scan_task_watch_output(&self, task: &Arc<BgTask>) {
2899 let mode = match task.state.lock() {
2900 Ok(state) => state.metadata.mode.clone(),
2901 Err(_) => return,
2902 };
2903 let mut stdout = (mode == BgMode::Pipes)
2904 .then(|| open_task_artifact(&task.paths, TaskArtifact::Stdout))
2905 .transpose()
2906 .ok()
2907 .flatten();
2908 let mut stderr = (mode == BgMode::Pipes)
2909 .then(|| open_task_artifact(&task.paths, TaskArtifact::Stderr))
2910 .transpose()
2911 .ok()
2912 .flatten();
2913 let mut pty = (mode == BgMode::Pty)
2914 .then(|| open_task_artifact(&task.paths, TaskArtifact::Pty))
2915 .transpose()
2916 .ok()
2917 .flatten();
2918 let mut matches = Vec::new();
2919 if let Ok(mut registry) = self.inner.watch_registry.lock() {
2920 match mode {
2921 BgMode::Pipes => {
2922 let (Some(stdout), Some(stderr)) = (stdout.as_mut(), stderr.as_mut()) else {
2923 return;
2924 };
2925 let stdout_key = format!("{}:stdout", task.task_id);
2926 let stderr_key = format!("{}:stderr", task.task_id);
2927 matches.extend(registry.scan_file_new_bytes(
2928 &stdout_key,
2929 &task.task_id,
2930 stdout,
2931 ));
2932 matches.extend(registry.scan_file_new_bytes(
2933 &stderr_key,
2934 &task.task_id,
2935 stderr,
2936 ));
2937 }
2938 BgMode::Pty => {
2939 let Some(pty) = pty.as_mut() else {
2940 return;
2941 };
2942 let pty_key = format!("{}:pty", task.task_id);
2943 matches.extend(registry.scan_file_new_bytes(&pty_key, &task.task_id, pty));
2944 }
2945 }
2946 }
2947 let (stdout_offset, stderr_offset, pty_offset) = self.watch_stream_cursors(&task.task_id);
2948 if matches.is_empty() {
2949 if self.task_has_watch_control(&task.task_id) {
2952 self.persist_task_watch_cursors(
2953 &task.session_id,
2954 &task.task_id,
2955 stdout_offset,
2956 stderr_offset,
2957 pty_offset,
2958 );
2959 }
2960 return;
2961 }
2962 for pattern_match in matches {
2963 self.persist_watch_match(
2964 &task.session_id,
2965 &task.task_id,
2966 &pattern_match,
2967 stdout_offset,
2968 stderr_offset,
2969 pty_offset,
2970 );
2971 self.emit_bash_pattern_match(&task.session_id, pattern_match);
2972 }
2973 self.persist_task_watch_cursors(
2974 &task.session_id,
2975 &task.task_id,
2976 stdout_offset,
2977 stderr_offset,
2978 pty_offset,
2979 );
2980 }
2981
2982 pub(crate) fn observed_status(
2983 &self,
2984 task_id: &str,
2985 session_id: &str,
2986 preview_bytes: usize,
2987 ) -> Option<BgTaskSnapshot> {
2988 validate_task_id(task_id).ok()?;
2989 let task = self.task_for_session(task_id, session_id)?;
2990 Some(self.snapshot_with_terminal_cache(&task, preview_bytes))
2991 }
2992
2993 pub fn status(
2994 &self,
2995 task_id: &str,
2996 session_id: &str,
2997 project_root: Option<&Path>,
2998 storage_dir: Option<&Path>,
2999 preview_bytes: usize,
3000 ) -> Option<BgTaskSnapshot> {
3001 validate_task_id(task_id).ok()?;
3002 let terminal_db_fallback_allowed = storage_dir
3003 .is_some_and(|storage_dir| task_bundle_is_absent(storage_dir, session_id, task_id));
3004 let mut task = self.task_for_session(task_id, session_id);
3005 if task.is_none() {
3006 if let Some(storage_dir) = storage_dir {
3007 let _ = if let Some(project_root) = project_root {
3008 self.replay_session_for_project(storage_dir, session_id, project_root)
3009 } else {
3010 self.replay_session(storage_dir, session_id)
3011 };
3012 task = self.task_for_session(task_id, session_id);
3013 }
3014 }
3015 let Some(task) = task else {
3016 if terminal_db_fallback_allowed {
3017 if let Some(snapshot) = storage_dir.and_then(|storage_dir| {
3018 self.terminal_db_status_for_session(session_id, task_id, storage_dir)
3019 }) {
3020 return Some(snapshot);
3021 }
3022 }
3023 return self.status_relaxed(
3024 task_id,
3025 session_id,
3026 project_root?,
3027 storage_dir?,
3028 preview_bytes,
3029 terminal_db_fallback_allowed,
3030 );
3031 };
3032 let _ = self.poll_task(&task);
3033 Some(self.snapshot_with_terminal_cache(&task, preview_bytes))
3034 }
3035
3036 fn status_relaxed_task(
3037 &self,
3038 task_id: &str,
3039 project_root: &Path,
3040 storage_dir: &Path,
3041 ) -> Option<Arc<BgTask>> {
3042 validate_task_id(task_id).ok()?;
3043 let canonical_project = canonicalized_path(project_root);
3044 match self.lookup_relaxed_task_from_db(task_id, project_root) {
3045 Some(Ok(Some(row))) => {
3046 let metadata = PersistedTask::from(row);
3047 if let Some(task) = self.task(task_id) {
3048 let matches_project = task
3049 .state
3050 .lock()
3051 .map(|state| {
3052 state
3053 .metadata
3054 .project_root
3055 .as_deref()
3056 .map(canonicalized_path)
3057 .as_deref()
3058 == Some(canonical_project.as_path())
3059 })
3060 .unwrap_or(false);
3061 return matches_project.then_some(task);
3062 }
3063 let resolved = resolve_task_layout(
3064 &session_tasks_dir(storage_dir, &metadata.session_id),
3065 &metadata.task_id,
3066 )
3067 .ok()?;
3068 let disk = read_task_at(&resolved).ok()?;
3069 if disk.task_id != metadata.task_id || disk.session_id != metadata.session_id {
3070 return None;
3071 }
3072 if self
3073 .insert_rehydrated_task(metadata, resolved.paths, true)
3074 .is_err()
3075 {
3076 return None;
3077 }
3078 return self.task(task_id);
3079 }
3080 Some(Ok(None)) => {
3081 crate::slog_info!(
3082 "bash task relaxed DB miss for {}; falling back to disk",
3083 task_id
3084 );
3085 }
3086 Some(Err(error)) => {
3087 crate::slog_warn!(
3088 "bash task relaxed DB lookup failed for {}; falling back to disk: {}",
3089 task_id,
3090 error
3091 );
3092 }
3093 None => {
3094 crate::slog_info!(
3095 "bash task relaxed DB unavailable for {}; falling back to disk",
3096 task_id
3097 );
3098 }
3099 }
3100 let root = storage_dir.join("bash-tasks");
3101 let entries = fs::read_dir(&root).ok()?;
3102 for entry in entries.flatten() {
3103 let dir = entry.path();
3104 if !dir.is_dir() {
3105 continue;
3106 }
3107 let resolved = match resolve_task_layout(&dir, task_id) {
3108 Ok(task) => task,
3109 Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue,
3110 Err(error) => {
3111 if self.db_has_live_process_for_task(task_id) {
3112 crate::slog_warn!(
3113 "refusing to quarantine unresolved live background task {task_id} during relaxed lookup: {error}"
3114 );
3115 continue;
3116 }
3117 crate::slog_warn!(
3118 "quarantining unresolved background task {task_id} during relaxed lookup: {error}"
3119 );
3120 let _ = quarantine_task_layout(storage_dir, &dir, task_id, "invalid");
3121 continue;
3122 }
3123 };
3124 let metadata = match read_task_at(&resolved) {
3125 Ok(metadata) => metadata,
3126 Err(error) => {
3127 if self.db_has_live_process_for_task(task_id) {
3128 crate::slog_warn!(
3129 "refusing to quarantine unreadable live background task {task_id} during relaxed lookup: {error}"
3130 );
3131 continue;
3132 }
3133 crate::slog_warn!(
3134 "quarantining invalid background task metadata {task_id} during relaxed lookup: {error}"
3135 );
3136 let _ = quarantine_task_layout(storage_dir, &dir, task_id, "invalid");
3137 continue;
3138 }
3139 };
3140 let metadata_project = metadata.project_root.as_deref().map(canonicalized_path);
3141 if metadata_project.as_deref() != Some(canonical_project.as_path()) {
3142 continue;
3143 }
3144 if let Some(task) = self.task(task_id) {
3145 let matches_project = task
3146 .state
3147 .lock()
3148 .map(|state| {
3149 state
3150 .metadata
3151 .project_root
3152 .as_deref()
3153 .map(canonicalized_path)
3154 .as_deref()
3155 == Some(canonical_project.as_path())
3156 })
3157 .unwrap_or(false);
3158 return matches_project.then_some(task);
3159 }
3160 if self
3161 .insert_rehydrated_task(metadata, resolved.paths, true)
3162 .is_err()
3163 {
3164 return None;
3165 }
3166 return self.task(task_id);
3167 }
3168 None
3169 }
3170
3171 fn lookup_relaxed_task_from_db(
3172 &self,
3173 task_id: &str,
3174 project_root: &Path,
3175 ) -> Option<Result<Option<BashTaskRow>, String>> {
3176 let pool = self
3177 .inner
3178 .db_pool
3179 .read()
3180 .ok()
3181 .and_then(|slot| slot.clone())?;
3182 let harness = self
3183 .inner
3184 .db_harness
3185 .read()
3186 .ok()
3187 .and_then(|slot| slot.clone())?;
3188 let conn = match pool.lock() {
3189 Ok(conn) => conn,
3190 Err(_) => return Some(Err("db mutex poisoned".to_string())),
3191 };
3192 let project_key = crate::path_identity::project_scope_key(project_root);
3193 Some(
3194 crate::db::bash_tasks::find_bash_task_for_project(
3195 &conn,
3196 &harness,
3197 &project_key,
3198 task_id,
3199 )
3200 .map_err(|error| error.to_string()),
3201 )
3202 }
3203
3204 pub(super) fn status_relaxed(
3205 &self,
3206 task_id: &str,
3207 _session_id: &str,
3208 project_root: &Path,
3209 storage_dir: &Path,
3210 preview_bytes: usize,
3211 allow_terminal_db_fallback: bool,
3212 ) -> Option<BgTaskSnapshot> {
3213 let fallback_row = if allow_terminal_db_fallback {
3214 self.lookup_relaxed_task_from_db(task_id, project_root)
3215 } else {
3216 None
3217 }
3218 .and_then(Result::ok)
3219 .flatten()
3220 .filter(|row| task_bundle_is_absent(storage_dir, &row.session_id, &row.task_id));
3221 if let Some(task) = self.status_relaxed_task(task_id, project_root, storage_dir) {
3222 let _ = self.poll_task(&task);
3223 return Some(self.snapshot_with_terminal_cache(&task, preview_bytes));
3224 }
3225 let row = fallback_row?;
3226 let metadata = PersistedTask::from(row.clone());
3227 metadata
3228 .status
3229 .is_terminal()
3230 .then(|| terminal_db_row_snapshot(row, metadata))
3231 }
3232
3233 pub fn kill_relaxed(
3234 &self,
3235 task_id: &str,
3236 project_root: &Path,
3237 storage_dir: &Path,
3238 ) -> Result<BgTaskSnapshot, String> {
3239 let task = self
3240 .status_relaxed_task(task_id, project_root, storage_dir)
3241 .ok_or_else(|| format!("background task not found: {task_id}"))?;
3242 self.kill_with_status(task_id, &task.session_id, BgTaskStatus::Killed)
3243 }
3244
3245 pub fn maybe_gc_persisted(&self, storage_dir: &Path) -> Result<usize, String> {
3246 #[cfg(test)]
3247 self.inner.persisted_gc_runs.fetch_add(1, Ordering::SeqCst);
3248 struct RecordThreadOnExit<'a>(&'a RegistryInner);
3253 impl Drop for RecordThreadOnExit<'_> {
3254 fn drop(&mut self) {
3255 *self
3256 .0
3257 .persisted_gc_thread
3258 .lock()
3259 .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(
3260 std::thread::current()
3261 .name()
3262 .unwrap_or("<unnamed>")
3263 .to_string(),
3264 );
3265 }
3266 }
3267 let _record_thread = RecordThreadOnExit(&self.inner);
3268
3269 let mut deleted = 0usize;
3270
3271 let root = storage_dir.join("bash-tasks");
3272 if root.exists() {
3273 let session_dirs = fs::read_dir(&root).map_err(|e| {
3274 format!(
3275 "failed to read background task root {}: {e}",
3276 root.display()
3277 )
3278 })?;
3279 for session_entry in session_dirs.flatten() {
3280 let session_dir = session_entry.path();
3281 if !session_dir.is_dir() {
3282 continue;
3283 }
3284 let (task_ids, invalid_entries) = match discover_task_ids(&session_dir) {
3285 Ok(discovery) => discovery,
3286 Err(error) => {
3287 crate::slog_warn!(
3288 "failed to discover background task session {}: {error}",
3289 session_dir.display()
3290 );
3291 continue;
3292 }
3293 };
3294 for entry in invalid_entries {
3295 let _ = quarantine_invalid_entry(storage_dir, &session_dir, &entry);
3296 }
3297 for task_id in task_ids {
3298 let resolved = match resolve_task_layout(&session_dir, &task_id) {
3299 Ok(task) => task,
3300 Err(error)
3305 if error.kind() == std::io::ErrorKind::NotFound
3306 && uninitialized_layout_is_recent(
3307 &session_dir,
3308 &task_id,
3309 Duration::from_secs(5 * 60),
3310 )
3311 .unwrap_or(true) =>
3312 {
3313 continue;
3314 }
3315 Err(error) => {
3316 if self.db_has_live_process_for_task(&task_id) {
3317 crate::slog_warn!(
3318 "refusing to quarantine unresolved live background task {task_id} during GC: {error}"
3319 );
3320 continue;
3321 }
3322 crate::slog_warn!(
3323 "quarantining unresolved background task {task_id}: {error}"
3324 );
3325 quarantine_task_layout(storage_dir, &session_dir, &task_id, "invalid")
3326 .map_err(|error| error.to_string())?;
3327 continue;
3328 }
3329 };
3330 if modified_within(&resolved.paths.json, PERSISTED_GC_GRACE) {
3331 continue;
3332 }
3333 let metadata = match read_task_at(&resolved) {
3334 Ok(metadata) => metadata,
3335 Err(error) => {
3336 if self.db_has_live_process_for_task(&task_id) {
3337 crate::slog_warn!(
3338 "refusing to quarantine unreadable live background task {task_id} during GC: {error}"
3339 );
3340 continue;
3341 }
3342 crate::slog_warn!(
3343 "quarantining corrupt background task metadata {task_id}: {error}"
3344 );
3345 quarantine_task_layout(storage_dir, &session_dir, &task_id, "corrupt")
3346 .map_err(|error| error.to_string())?;
3347 continue;
3348 }
3349 };
3350 if !(metadata.status.is_terminal() && metadata.completion_delivered) {
3351 continue;
3352 }
3353 if Self::persisted_task_process_is_alive(&metadata)
3354 || self.db_has_live_process_for_task(&task_id)
3355 {
3356 crate::slog_warn!(
3357 "refusing to delete terminal background task bundle {task_id}: recorded process is still alive"
3358 );
3359 continue;
3360 }
3361 match delete_task_bundle(&resolved.paths) {
3362 Ok(()) => {
3363 self.delete_gc_task_from_db(&metadata);
3364 self.evaluate_erased_watch_targets();
3365 deleted += 1;
3366 log::debug!(
3367 "deleted persisted background task bundle {}",
3368 metadata.task_id
3369 );
3370 }
3371 Err(error) => {
3372 crate::slog_warn!(
3373 "failed to delete background task bundle {}: {error}",
3374 metadata.task_id
3375 );
3376 }
3377 }
3378 }
3379 }
3380 }
3381 gc_quarantine(storage_dir);
3382 Ok(deleted)
3383 }
3384
3385 pub fn list(&self, preview_bytes: usize) -> Vec<BgTaskSnapshot> {
3386 let tasks = self
3387 .inner
3388 .tasks
3389 .lock()
3390 .map(|tasks| tasks.values().cloned().collect::<Vec<_>>())
3391 .unwrap_or_default();
3392 tasks
3393 .into_iter()
3394 .map(|task| {
3395 let _ = self.poll_task(&task);
3396 self.snapshot_with_terminal_cache(&task, preview_bytes)
3397 })
3398 .collect()
3399 }
3400
3401 fn maybe_compress_snapshot(&self, task: &Arc<BgTask>, snapshot: &mut BgTaskSnapshot) {
3407 if !snapshot.info.status.is_terminal() || snapshot.info.mode == BgMode::Pty {
3408 return;
3409 }
3410 if let Some(cache) = self.ensure_terminal_output_cache(task) {
3411 let mut output_preview = cache.output_preview.clone();
3412 let envelope = append_bash_output_envelope(&cache, &mut output_preview);
3413 snapshot.output_preview = output_preview;
3414 snapshot.bash_output_list_envelope = envelope;
3415 snapshot.output_truncated = cache.output_truncated;
3416 }
3417 }
3418
3419 pub fn kill(&self, task_id: &str, session_id: &str) -> Result<BgTaskSnapshot, String> {
3420 self.kill_with_status(task_id, session_id, BgTaskStatus::Killed)
3421 }
3422
3423 pub fn kill_running_tasks_for_root(&self, project_root: &Path) -> usize {
3433 let canonical_root = canonicalized_path(project_root);
3434 let targets = self
3435 .inner
3436 .tasks
3437 .lock()
3438 .map(|tasks| {
3439 tasks
3440 .values()
3441 .filter_map(|task| {
3442 let state = task.state.lock().ok()?;
3443 let status = &state.metadata.status;
3444 let running = matches!(status, BgTaskStatus::Running)
3445 || (state.metadata.mode == BgMode::Pty
3446 && matches!(status, BgTaskStatus::Killing));
3447 if !running {
3448 return None;
3449 }
3450 let task_root = state
3451 .metadata
3452 .project_root
3453 .as_deref()
3454 .unwrap_or(&state.metadata.workdir);
3455 (canonicalized_path(task_root) == canonical_root)
3456 .then(|| (task.task_id.clone(), task.session_id.clone()))
3457 })
3458 .collect::<Vec<_>>()
3459 })
3460 .unwrap_or_default();
3461
3462 let mut killed = 0;
3463 for (task_id, session_id) in targets {
3464 match self.kill_with_status_reason(
3465 &task_id,
3466 &session_id,
3467 BgTaskStatus::Killed,
3468 Some(ROOT_RECLAIMED_REASON.to_string()),
3469 ) {
3470 Ok(_) => killed += 1,
3471 Err(error) => crate::slog_warn!(
3472 "failed to terminate background task {task_id} for reclaimed root {}: {error}",
3473 project_root.display()
3474 ),
3475 }
3476 }
3477 killed
3478 }
3479
3480 pub fn promote(&self, task_id: &str, session_id: &str) -> Result<bool, String> {
3481 let task = self
3482 .task_for_session(task_id, session_id)
3483 .ok_or_else(|| format!("background task not found: {task_id}"))?;
3484 let terminal_after_promote = {
3485 let mut state = task
3486 .state
3487 .lock()
3488 .map_err(|_| "background task lock poisoned".to_string())?;
3489 let updated = self
3490 .update_task_metadata(&task.paths, |metadata| {
3491 metadata.notify_on_completion = true;
3492 metadata.completion_delivered = false;
3493 })
3494 .map_err(|e| format!("failed to promote background task: {e}"))?;
3495 state.metadata = updated;
3496 state.metadata.status.is_terminal()
3497 };
3498 if terminal_after_promote {
3499 self.post_terminal_transition(&task, true)?;
3500 }
3501 Ok(true)
3502 }
3503
3504 pub(crate) fn kill_for_timeout(&self, task_id: &str, session_id: &str) -> Result<(), String> {
3505 self.kill_with_status(task_id, session_id, BgTaskStatus::TimedOut)
3506 .map(|_| ())
3507 }
3508
3509 pub fn cleanup_finished(&self, older_than: Duration) {
3510 let cutoff = Instant::now().checked_sub(older_than);
3511 let removable_paths: Vec<(String, TaskPaths)> =
3512 if let Ok(mut tasks) = self.inner.tasks.lock() {
3513 let removable = tasks
3514 .iter()
3515 .filter_map(|(task_id, task)| {
3516 let delivered_terminal = task
3517 .state
3518 .lock()
3519 .map(|state| {
3520 state.metadata.status.is_terminal()
3521 && state.metadata.completion_delivered
3522 })
3523 .unwrap_or(false);
3524 if !delivered_terminal {
3525 return None;
3526 }
3527
3528 let terminal_at = task.terminal_at.lock().ok().and_then(|at| *at);
3529 let expired = match (terminal_at, cutoff) {
3530 (Some(terminal_at), Some(cutoff)) => terminal_at <= cutoff,
3531 (Some(_), None) => true,
3532 (None, _) => false,
3533 };
3534 expired.then(|| task_id.clone())
3535 })
3536 .collect::<Vec<_>>();
3537
3538 removable
3539 .into_iter()
3540 .filter_map(|task_id| {
3541 tasks
3542 .remove(&task_id)
3543 .map(|task| (task_id, task.paths.clone()))
3544 })
3545 .collect()
3546 } else {
3547 Vec::new()
3548 };
3549
3550 for (task_id, paths) in removable_paths {
3551 match delete_task_bundle(&paths) {
3552 Ok(()) => log::debug!("deleted persisted background task bundle {task_id}"),
3553 Err(error) => crate::slog_warn!(
3554 "failed to delete persisted background task bundle {task_id}: {error}"
3555 ),
3556 }
3557 }
3558 }
3559
3560 pub fn drain_completions(&self) -> Vec<BgCompletion> {
3561 self.drain_completions_for_session(None)
3562 }
3563
3564 pub fn drain_completions_for_session(&self, session_id: Option<&str>) -> Vec<BgCompletion> {
3565 if let Some(session_id) = session_id {
3566 let pending_matches = self.pending_pattern_matches_for_session(session_id);
3567 if let Ok(sender) = self
3568 .inner
3569 .progress_sender
3570 .lock()
3571 .map(|sender| sender.clone())
3572 {
3573 if let Some(sender) = sender.as_ref() {
3574 for pattern_match in pending_matches {
3575 sender(PushFrame::BashPatternMatch(pattern_match));
3576 }
3577 }
3578 }
3579 }
3580 let completions = match self.inner.completions.lock() {
3581 Ok(completions) => completions,
3582 Err(_) => return Vec::new(),
3583 };
3584
3585 completions
3586 .iter()
3587 .filter(|completion| completion_matches_session(completion, session_id))
3588 .cloned()
3589 .collect()
3590 }
3591
3592 pub fn has_completions_for_session(&self, session_id: Option<&str>) -> bool {
3593 match self.inner.completions.lock() {
3594 Ok(completions) => completions
3595 .iter()
3596 .any(|completion| completion_matches_session(completion, session_id)),
3597 Err(_) => true,
3601 }
3602 }
3603
3604 pub fn unacked_wake_keys(&self) -> HashSet<String> {
3605 let mut keys = self
3606 .inner
3607 .completions
3608 .lock()
3609 .map(|completions| {
3610 completions
3611 .iter()
3612 .map(|completion| {
3613 format!(
3614 "completion\0{}\0{}",
3615 completion.session_id, completion.task_id
3616 )
3617 })
3618 .collect::<HashSet<_>>()
3619 })
3620 .unwrap_or_default();
3621 let Some((harness, pool)) = self.db_harness_and_pool() else {
3622 return keys;
3623 };
3624 let Ok(conn) = pool.lock() else {
3625 return keys;
3626 };
3627 if let Ok(rows) = crate::db::bash_watches::list_bash_pattern_watches(&conn, &harness) {
3628 keys.extend(rows.into_iter().filter(|row| row.pending_match).map(|row| {
3629 format!(
3630 "match\0{}\0{}\0{}",
3631 row.session_id, row.task_id, row.watch_id
3632 )
3633 }));
3634 }
3635 keys
3636 }
3637
3638 pub fn unacked_wake_count_for_session(&self, session_id: Option<&str>) -> usize {
3639 let completion_count = self
3640 .inner
3641 .completions
3642 .lock()
3643 .map(|completions| {
3644 completions
3645 .iter()
3646 .filter(|completion| completion_matches_session(completion, session_id))
3647 .count()
3648 })
3649 .unwrap_or(1);
3650 let Some((harness, pool)) = self.db_harness_and_pool() else {
3651 return completion_count;
3652 };
3653 let Ok(conn) = pool.lock() else {
3654 return completion_count.saturating_add(1);
3655 };
3656 let pending_matches = match session_id {
3657 Some(session_id) => {
3658 crate::db::bash_watches::count_pending_bash_pattern_watches_for_session(
3659 &conn, &harness, session_id,
3660 )
3661 }
3662 None => crate::db::bash_watches::count_pending_bash_pattern_watches(&conn, &harness),
3663 };
3664 completion_count.saturating_add(pending_matches.unwrap_or(1))
3665 }
3666
3667 pub fn has_unacked_wakes_for_session(&self, session_id: &str) -> bool {
3668 self.unacked_wake_count_for_session(Some(session_id)) > 0
3669 }
3670
3671 pub fn stuck_pending_watches_for_session(
3672 &self,
3673 session_id: &str,
3674 older_than: Duration,
3675 ) -> Vec<(String, String, u64)> {
3676 let Some((harness, pool)) = self.db_harness_and_pool() else {
3677 return Vec::new();
3678 };
3679 let Ok(conn) = pool.lock() else {
3680 return Vec::new();
3681 };
3682 let now = unix_millis();
3683 let minimum_age_ms = older_than.as_millis().min(u128::from(u64::MAX)) as u64;
3684 crate::db::bash_watches::list_bash_pattern_watches_for_session(&conn, &harness, session_id)
3685 .unwrap_or_default()
3686 .into_iter()
3687 .filter(|row| row.pending_match && self.task(&row.task_id).is_some())
3688 .filter_map(|row| {
3689 let created_at = u64::try_from(row.created_at).ok()?;
3690 let age_ms = now.saturating_sub(created_at);
3691 (age_ms >= minimum_age_ms).then_some((row.task_id, row.watch_id, age_ms))
3692 })
3693 .collect()
3694 }
3695
3696 pub fn ack_completions_for_session(
3697 &self,
3698 session_id: Option<&str>,
3699 task_ids: &[String],
3700 ) -> Vec<String> {
3701 if task_ids.is_empty() {
3702 return Vec::new();
3703 }
3704 let requested_task_ids = task_ids.iter().map(String::as_str).collect::<HashSet<_>>();
3705 let mut completion_sessions = HashMap::new();
3706 if let Ok(mut completions) = self.inner.completions.lock() {
3707 completions.retain(|completion| {
3708 let session_matches = session_id
3709 .map(|session_id| completion.session_id == session_id)
3710 .unwrap_or(true);
3711 if session_matches && requested_task_ids.contains(completion.task_id.as_str()) {
3712 completion_sessions
3713 .insert(completion.task_id.clone(), completion.session_id.clone());
3714 false
3715 } else {
3716 true
3717 }
3718 });
3719 }
3720
3721 let mut delivered = Vec::new();
3722 for task_id in task_ids {
3723 if self.has_erased_watch_reference(task_id) {
3724 if let Some((harness, pool)) = self.db_harness_and_pool() {
3725 if let Ok(conn) = pool.lock() {
3726 if let Ok(rows) =
3727 crate::db::bash_watches::list_bash_pattern_watches_by_task_id(
3728 &conn, &harness, task_id,
3729 )
3730 {
3731 for row in rows {
3732 let _ = crate::db::bash_watches::delete_bash_pattern_watch(
3733 &conn,
3734 &harness,
3735 &row.session_id,
3736 task_id,
3737 &row.watch_id,
3738 );
3739 }
3740 }
3741 }
3742 }
3743 self.clear_task_watch_state(task_id);
3744 delivered.push(task_id.clone());
3747 continue;
3748 }
3749 let task = if let Some(session_id) = session_id {
3750 self.task_for_session(task_id, session_id).or_else(|| {
3751 completion_sessions
3752 .contains_key(task_id)
3753 .then(|| self.task(task_id))
3754 .flatten()
3755 })
3756 } else if let Some(completion_session_id) = completion_sessions.get(task_id) {
3757 self.task_for_session(task_id, completion_session_id)
3758 .or_else(|| self.task(task_id))
3759 } else {
3760 self.task(task_id)
3761 };
3762 if let Some(task) = task {
3763 let terminal = task
3764 .state
3765 .lock()
3766 .map(|state| state.metadata.status.is_terminal())
3767 .unwrap_or(false);
3768 self.ack_persisted_watches_for_task(&task.session_id, task_id, terminal);
3772 if terminal {
3773 self.clear_task_watch_state(task_id);
3774 if task.set_completion_delivered(true, self).is_ok() {
3775 delivered.push(task_id.clone());
3776 }
3777 } else {
3778 self.sync_memory_watches_from_persistence(task_id);
3781 delivered.push(task_id.clone());
3782 }
3783 } else if let Some(session_id) = session_id {
3784 let terminal = self
3788 .persisted_task_is_terminal(session_id, task_id)
3789 .unwrap_or(true);
3790 self.ack_persisted_watches_for_task(session_id, task_id, terminal);
3791 if terminal {
3792 self.mark_persisted_completion_delivered(session_id, task_id);
3793 }
3794 delivered.push(task_id.clone());
3795 }
3796 }
3797
3798 delivered
3799 }
3800
3801 fn persisted_task_is_terminal(&self, session_id: &str, task_id: &str) -> Option<bool> {
3802 let (harness, pool) = self.db_harness_and_pool()?;
3803 let conn = pool.lock().ok()?;
3804 let row = crate::db::bash_tasks::get_bash_task(&conn, &harness, session_id, task_id)
3805 .ok()
3806 .flatten()?;
3807 Some(matches!(
3808 row.status.as_str(),
3809 "completed" | "failed" | "killed" | "timed_out" | "fate_unknown"
3810 ))
3811 }
3812
3813 fn mark_persisted_completion_delivered(&self, session_id: &str, task_id: &str) {
3814 let Some((harness, pool)) = self.db_harness_and_pool() else {
3815 return;
3816 };
3817 let Ok(conn) = pool.lock() else {
3818 return;
3819 };
3820 let _ = conn.execute(
3821 "UPDATE bash_tasks SET completion_delivered = 1
3822 WHERE harness = ?1 AND session_id = ?2 AND task_id = ?3
3823 AND status IN ('completed', 'failed', 'killed', 'timed_out', 'fate_unknown')",
3824 rusqlite::params![harness, session_id, task_id],
3825 );
3826 }
3827
3828 fn sync_memory_watches_from_persistence(&self, task_id: &str) {
3829 let Some((harness, pool)) = self.db_harness_and_pool() else {
3830 return;
3831 };
3832 let session_id = match self.task(task_id) {
3833 Some(task) => task.session_id.clone(),
3834 None => return,
3835 };
3836 let Ok(conn) = pool.lock() else {
3837 return;
3838 };
3839 let Ok(rows) = crate::db::bash_watches::list_bash_pattern_watches_for_task(
3840 &conn,
3841 &harness,
3842 &session_id,
3843 task_id,
3844 ) else {
3845 return;
3846 };
3847 let has_pending_match = rows.iter().any(|row| row.pending_match);
3848 let remaining: HashSet<String> = rows.into_iter().map(|row| row.watch_id).collect();
3849 if let Ok(mut registry) = self.inner.watch_registry.lock() {
3850 registry.reconcile_watch_ids(task_id, &remaining, has_pending_match);
3851 }
3852 }
3853
3854 pub fn pending_completions_for_session(&self, session_id: &str) -> Vec<BgCompletion> {
3855 self.inner
3856 .completions
3857 .lock()
3858 .map(|completions| {
3859 completions
3860 .iter()
3861 .filter(|completion| completion.session_id == session_id)
3862 .cloned()
3863 .collect()
3864 })
3865 .unwrap_or_default()
3866 }
3867
3868 fn remove_pending_completion(&self, task_id: &str) -> Option<BgCompletion> {
3869 let mut completions = self.inner.completions.lock().ok()?;
3870 let idx = completions
3871 .iter()
3872 .position(|completion| completion.task_id == task_id)?;
3873 completions.remove(idx)
3874 }
3875
3876 fn completion_snapshot_for_task(&self, task: &Arc<BgTask>) -> Option<BgCompletion> {
3877 let snapshot = self.snapshot_with_terminal_cache(task, RUNNING_OUTPUT_PREVIEW_BYTES);
3878 if !snapshot.info.status.is_terminal() {
3879 return None;
3880 }
3881 let (mut output_preview, output_truncated, cache) = if snapshot.info.mode == BgMode::Pty {
3882 (String::new(), false, None)
3883 } else {
3884 self.ensure_terminal_output_cache(task)
3885 .map(|cache| {
3886 let (preview, truncated) =
3887 completion_preview_for_cache(&cache, snapshot.exit_code);
3888 (preview, truncated, Some(cache))
3889 })
3890 .unwrap_or_else(|| (String::new(), false, None))
3891 };
3892 let bash_output_list_envelope = cache
3893 .as_ref()
3894 .and_then(|cache| append_bash_output_envelope(cache, &mut output_preview));
3895 Some(BgCompletion {
3896 task_id: snapshot.info.task_id,
3897 session_id: task.session_id.clone(),
3898 status: snapshot.info.status,
3899 exit_code: snapshot.exit_code,
3900 command: snapshot.info.command,
3901 output_preview,
3902 bash_output_list_envelope,
3903 output_truncated,
3904 original_tokens: None,
3905 compressed_tokens: None,
3906 tokens_skipped: false,
3907 status_reason: snapshot.info.status_reason,
3908 live_descendants: snapshot.live_descendants,
3909 live_descendants_omitted: snapshot.live_descendants_omitted,
3910 live_descendants_summary: snapshot.live_descendants_summary,
3911 })
3912 }
3913
3914 pub fn detach(&self) {
3915 self.inner.shutdown.store(true, Ordering::SeqCst);
3916 if let Ok(mut tasks) = self.inner.tasks.lock() {
3917 for task in tasks.values() {
3918 if let Ok(mut state) = task.state.lock() {
3919 match &mut state.runtime {
3920 TaskRuntime::Piped(child) => *child = None,
3921 TaskRuntime::Pty(runtime) => *runtime = None,
3922 }
3923 state.detached = true;
3924 }
3925 }
3926 tasks.clear();
3927 }
3928 }
3929
3930 pub fn shutdown(&self) {
3931 let tasks = self
3932 .inner
3933 .tasks
3934 .lock()
3935 .map(|tasks| {
3936 tasks
3937 .values()
3938 .map(|task| (task.task_id.clone(), task.session_id.clone()))
3939 .collect::<Vec<_>>()
3940 })
3941 .unwrap_or_default();
3942 for (task_id, session_id) in tasks {
3943 let _ = self.kill(&task_id, &session_id);
3944 }
3945 }
3946
3947 pub(crate) fn poll_task(&self, task: &Arc<BgTask>) -> Result<(), String> {
3948 if let Ok(state) = task.state.lock() {
3949 if let TaskRuntime::Pty(Some(pty)) = &state.runtime {
3950 if !pty.exit_observed.load(Ordering::SeqCst) {
3958 return Ok(());
3959 }
3960 }
3961 }
3962 let marker = match read_exit_marker(&task.paths) {
3963 Ok(Some(marker)) => marker,
3964 Ok(None) => return Ok(()),
3965 Err(error) => return Err(format!("failed to read exit marker: {error}")),
3966 };
3967 self.finalize_from_marker(task, marker, None)
3968 }
3969
3970 pub(crate) fn reap_child(&self, task: &Arc<BgTask>) {
3971 let mut needs_completion = false;
3972 {
3973 let Ok(mut state) = task.state.lock() else {
3974 return;
3975 };
3976 match &mut state.runtime {
3977 TaskRuntime::Piped(child_slot) => {
3978 if let Some(child) = child_slot.as_mut() {
3979 if let Ok(Some(status)) = child.try_wait() {
3980 *child_slot = None;
3981 state.detached = true;
3982 state.child_exit_observed = true;
3983 if let Some(handles) = state.io_handles.as_mut() {
3984 if handles.artifact_len(TaskArtifact::Exit).unwrap_or(1) == 0 {
3985 let marker = status
3986 .code()
3987 .map(|code| code.to_string())
3988 .unwrap_or_else(|| "1".to_string());
3989 let _ = handles.write(TaskArtifact::Exit, marker.as_bytes());
3990 }
3991 }
3992 }
3993 } else if state.detached {
3994 let child_known_dead = state.child_exit_observed
3995 || state
3996 .metadata
3997 .child_pid
3998 .is_some_and(|pid| !is_process_alive(pid));
3999 if child_known_dead {
4000 needs_completion =
4001 self.fail_without_exit_marker_if_needed(task, &mut state);
4002 }
4003 }
4004 }
4005 TaskRuntime::Pty(Some(pty)) => {
4006 if pty.exit_observed.load(Ordering::SeqCst) {
4007 drop(state);
4008 let _ = self.poll_task(task);
4009 return;
4010 }
4011 }
4012 TaskRuntime::Pty(None) => {}
4013 }
4014 }
4015 if needs_completion {
4016 let _ = self.post_terminal_transition(task, true);
4017 }
4018 }
4019
4020 fn fail_without_exit_marker_if_needed(
4021 &self,
4022 task: &Arc<BgTask>,
4023 state: &mut BgTaskState,
4024 ) -> bool {
4025 if state.metadata.status.is_terminal() {
4026 return false;
4027 }
4028 if matches!(read_exit_marker(&task.paths), Ok(Some(_))) {
4029 return false;
4030 }
4031 let child_exit_observed = state.child_exit_observed;
4032 let updated = self.update_task_metadata(&task.paths, |metadata| {
4033 let (status, reason) = if child_exit_observed {
4034 (
4035 BgTaskStatus::Failed,
4036 "process exited without exit marker".to_string(),
4037 )
4038 } else {
4039 (
4040 BgTaskStatus::FateUnknown,
4041 restart_fate_unknown_reason(metadata, &task.paths),
4042 )
4043 };
4044 metadata.mark_terminal(status, None, Some(reason));
4045 });
4046 if let Ok(metadata) = updated {
4047 state.pending_terminal_override = None;
4048 state.metadata = metadata;
4049 task.mark_terminal_now();
4050 return true;
4051 }
4052 false
4053 }
4054
4055 pub(crate) fn running_tasks(&self) -> Vec<Arc<BgTask>> {
4056 self.inner
4057 .tasks
4058 .lock()
4059 .map(|tasks| {
4060 tasks
4061 .values()
4062 .filter(|task| task.is_running())
4063 .cloned()
4064 .collect()
4065 })
4066 .unwrap_or_default()
4067 }
4068
4069 fn insert_rehydrated_task(
4070 &self,
4071 metadata: PersistedTask,
4072 paths: TaskPaths,
4073 detached: bool,
4074 ) -> Result<(), String> {
4075 let task_id = metadata.task_id.clone();
4076 let session_id = metadata.session_id.clone();
4077 let started = started_instant_from_unix_millis(metadata.started_at);
4078 let suppress_replayed_running_reminder = metadata.status == BgTaskStatus::Running;
4079 let mode = metadata.mode.clone();
4080 let task = Arc::new(BgTask {
4081 task_id: task_id.clone(),
4082 session_id,
4083 paths: paths.clone(),
4084 artifact_root: canonical_artifact_root(&paths),
4085 started,
4086 last_reminder_at: Mutex::new(suppress_replayed_running_reminder.then(Instant::now)),
4087 terminal_at: Mutex::new(metadata.status.is_terminal().then(Instant::now)),
4088 state: Mutex::new(BgTaskState {
4089 metadata,
4090 runtime: if mode == BgMode::Pty {
4091 TaskRuntime::Pty(None)
4092 } else {
4093 TaskRuntime::Piped(None)
4094 },
4095 io_handles: None,
4096 detached,
4097 child_exit_observed: false,
4104 descendant_sampling_started: false,
4105 buffer: BgBuffer::registered(&paths, mode.clone()),
4106 terminal_output_cache: None,
4107 pending_terminal_override: None,
4108 }),
4109 });
4110 self.record_live_delivery_session(&task.session_id);
4111 self.inner
4112 .tasks
4113 .lock()
4114 .map_err(|_| "background task registry lock poisoned".to_string())?
4115 .insert(task_id.clone(), Arc::clone(&task));
4116 self.rearm_persisted_watches(&task);
4120 Ok(())
4121 }
4122
4123 fn rearm_persisted_watches(&self, task: &Arc<BgTask>) {
4124 let Some((harness, pool)) = self.db_harness_and_pool() else {
4125 return;
4126 };
4127 let rows = {
4128 let Ok(conn) = pool.lock() else {
4129 return;
4130 };
4131 match crate::db::bash_watches::list_bash_pattern_watches_for_task(
4132 &conn,
4133 &harness,
4134 &task.session_id,
4135 &task.task_id,
4136 ) {
4137 Ok(rows) if !rows.is_empty() => rows,
4138 _ => return,
4139 }
4140 };
4141
4142 let mode = match task.state.lock() {
4143 Ok(state) => state.metadata.mode.clone(),
4144 Err(_) => return,
4145 };
4146 let terminal = task
4147 .state
4148 .lock()
4149 .map(|state| state.metadata.status.is_terminal())
4150 .unwrap_or(false);
4151 let completion_delivered = task
4152 .state
4153 .lock()
4154 .map(|state| state.metadata.completion_delivered)
4155 .unwrap_or(true);
4156
4157 let mut stdout = (mode == BgMode::Pipes)
4158 .then(|| open_task_artifact(&task.paths, TaskArtifact::Stdout))
4159 .transpose()
4160 .ok()
4161 .flatten();
4162 let mut stderr = (mode == BgMode::Pipes)
4163 .then(|| open_task_artifact(&task.paths, TaskArtifact::Stderr))
4164 .transpose()
4165 .ok()
4166 .flatten();
4167 let mut pty = (mode == BgMode::Pty)
4168 .then(|| open_task_artifact(&task.paths, TaskArtifact::Pty))
4169 .transpose()
4170 .ok()
4171 .flatten();
4172
4173 let mut pending_to_emit = Vec::new();
4174 let mut gap_matches = Vec::new();
4175 {
4176 let Ok(mut registry) = self.inner.watch_registry.lock() else {
4177 return;
4178 };
4179 let stdout_key = format!("{}:stdout", task.task_id);
4180 let stderr_key = format!("{}:stderr", task.task_id);
4181 let pty_key = format!("{}:pty", task.task_id);
4182
4183 let first = &rows[0];
4185 match mode {
4186 BgMode::Pipes => {
4187 registry.set_file_cursor(&stdout_key, first.stdout_offset.max(0) as u64);
4188 registry.set_file_cursor(&stderr_key, first.stderr_offset.max(0) as u64);
4189 }
4190 BgMode::Pty => {
4191 registry.set_file_cursor(&pty_key, first.pty_offset.max(0) as u64);
4192 }
4193 }
4194
4195 for row in &rows {
4196 let Ok(pattern) = WatchPattern::from_persisted(&row.pattern_kind, &row.pattern)
4197 else {
4198 crate::slog_warn!(
4199 "skipping unreadable persisted watch {}/{}",
4200 row.task_id,
4201 row.watch_id
4202 );
4203 continue;
4204 };
4205 if let Err(error) = registry.restore(
4206 row.watch_id.clone(),
4207 row.task_id.clone(),
4208 pattern,
4209 row.once,
4210 row.scanning,
4211 ) {
4212 crate::slog_warn!(
4213 "failed to restore watch {}/{}: {error}",
4214 row.task_id,
4215 row.watch_id
4216 );
4217 continue;
4218 }
4219 if row.pending_match {
4220 if let (Some(match_text), Some(match_offset), Some(context)) = (
4221 row.match_text.clone(),
4222 row.match_offset,
4223 row.match_context.clone(),
4224 ) {
4225 pending_to_emit.push(PatternMatch {
4226 watch_id: row.watch_id.clone(),
4227 task_id: row.task_id.clone(),
4228 match_text,
4229 match_offset: match_offset.max(0) as u64,
4230 context,
4231 once: row.once,
4232 });
4233 }
4234 }
4235 }
4236
4237 let should_gap_scan =
4241 rows.iter().any(|row| row.scanning) && !pending_to_emit.iter().any(|m| m.once);
4242 if should_gap_scan {
4243 match mode {
4244 BgMode::Pipes => {
4245 if let (Some(stdout), Some(stderr)) = (stdout.as_mut(), stderr.as_mut()) {
4246 gap_matches.extend(registry.scan_file_new_bytes(
4247 &stdout_key,
4248 &task.task_id,
4249 stdout,
4250 ));
4251 gap_matches.extend(registry.scan_file_new_bytes(
4252 &stderr_key,
4253 &task.task_id,
4254 stderr,
4255 ));
4256 }
4257 }
4258 BgMode::Pty => {
4259 if let Some(pty) = pty.as_mut() {
4260 gap_matches.extend(registry.scan_file_new_bytes(
4261 &pty_key,
4262 &task.task_id,
4263 pty,
4264 ));
4265 }
4266 }
4267 }
4268 }
4269 }
4270
4271 let (stdout_offset, stderr_offset, pty_offset) = self.watch_stream_cursors(&task.task_id);
4272 for pattern_match in &gap_matches {
4273 self.persist_watch_match(
4274 &task.session_id,
4275 &task.task_id,
4276 pattern_match,
4277 stdout_offset,
4278 stderr_offset,
4279 pty_offset,
4280 );
4281 }
4282 if !gap_matches.is_empty() || rows.iter().any(|row| row.scanning) {
4283 self.persist_task_watch_cursors(
4284 &task.session_id,
4285 &task.task_id,
4286 stdout_offset,
4287 stderr_offset,
4288 pty_offset,
4289 );
4290 }
4291
4292 let emitted_pending = !pending_to_emit.is_empty();
4294 let to_emit = if emitted_pending {
4295 pending_to_emit
4296 } else {
4297 gap_matches
4298 };
4299 for pattern_match in to_emit {
4300 self.emit_bash_pattern_match(&task.session_id, pattern_match);
4301 }
4302
4303 if !terminal {
4304 return;
4305 }
4306
4307 let (watch_controlled, watch_matched) = self.task_watch_state(&task.task_id);
4308 if !watch_controlled {
4309 return;
4310 }
4311 if watch_matched {
4312 let _ = self.remove_pending_completion(&task.task_id);
4315 return;
4316 }
4317 if completion_delivered {
4318 self.clear_task_watch_state(&task.task_id);
4320 self.delete_persisted_watches_for_task(&task.session_id, &task.task_id);
4321 return;
4322 }
4323 if let Some(completion) = self
4324 .remove_pending_completion(&task.task_id)
4325 .or_else(|| self.completion_snapshot_for_task(task))
4326 {
4327 self.record_bash_watch_exit(&completion, true);
4328 }
4329 self.clear_task_watch_state(&task.task_id);
4331 }
4332
4333 fn kill_with_status(
4334 &self,
4335 task_id: &str,
4336 session_id: &str,
4337 terminal_status: BgTaskStatus,
4338 ) -> Result<BgTaskSnapshot, String> {
4339 self.kill_with_status_reason(task_id, session_id, terminal_status, None)
4340 }
4341
4342 fn kill_with_status_reason(
4343 &self,
4344 task_id: &str,
4345 session_id: &str,
4346 terminal_status: BgTaskStatus,
4347 reason: Option<String>,
4348 ) -> Result<BgTaskSnapshot, String> {
4349 let task = self
4350 .task_for_session(task_id, session_id)
4351 .ok_or_else(|| format!("background task not found: {task_id}"))?;
4352 let mut terminalized = false;
4353 #[cfg_attr(not(unix), allow(unused_mut))]
4354 let mut kill_signaled = false;
4355 #[cfg_attr(not(unix), allow(unused_mut))]
4356 let mut kill_reached = 0;
4357
4358 {
4359 let mut state = task
4360 .state
4361 .lock()
4362 .map_err(|_| "background task lock poisoned".to_string())?;
4363 if state.metadata.status.is_terminal() {
4364 state.pending_terminal_override = None;
4365 #[cfg(unix)]
4366 let live_count = state
4367 .metadata
4368 .live_descendants
4369 .as_ref()
4370 .map(Vec::len)
4371 .unwrap_or(0)
4372 + state.metadata.live_descendants_omitted;
4373 #[cfg(unix)]
4374 if live_count > 0 {
4375 if let Some(pgid) = state.metadata.pgid {
4376 kill_signaled = true;
4377 kill_reached = live_count;
4378 terminate_pgid(pgid, None);
4379 let sample = live_process_group_members(pgid);
4380 state.metadata.live_descendants =
4381 sample.as_ref().map(|(members, _)| members.clone());
4382 state.metadata.live_descendants_omitted =
4383 sample.as_ref().map(|(_, omitted)| *omitted).unwrap_or(0);
4384 self.persist_task(&task.paths, &state.metadata)
4385 .map_err(|error| {
4386 format!("failed to persist post-kill descendant sample: {error}")
4387 })?;
4388 }
4389 }
4390 } else if let Ok(Some(marker)) = read_exit_marker(&task.paths) {
4391 state.metadata =
4392 terminal_metadata_from_marker(state.metadata.clone(), marker, reason.clone());
4393
4394 state.pending_terminal_override = None;
4395 task.mark_terminal_now();
4396 match &mut state.runtime {
4397 TaskRuntime::Piped(child_slot) => reap_piped_child(child_slot),
4404 TaskRuntime::Pty(runtime) => *runtime = None,
4405 }
4406 state.detached = true;
4407 self.persist_task(&task.paths, &state.metadata)
4408 .map_err(|e| format!("failed to persist terminal state: {e}"))?;
4409 terminalized = true;
4410 } else {
4411 let was_already_killing = state.metadata.status == BgTaskStatus::Killing;
4412 if !was_already_killing {
4413 state.metadata.status = BgTaskStatus::Killing;
4414 }
4415 if reason.is_some() {
4416 state.metadata.status_reason = reason.clone();
4417 }
4418 if !was_already_killing || reason.is_some() {
4419 self.persist_task(&task.paths, &state.metadata)
4420 .map_err(|e| format!("failed to persist killing state: {e}"))?;
4421 }
4422
4423 #[cfg(unix)]
4424 let pgid = state.metadata.pgid;
4425 #[cfg(windows)]
4426 let child_pid = state.metadata.child_pid;
4427 if !was_already_killing
4428 && state.metadata.mode == BgMode::Pty
4429 && terminal_status == BgTaskStatus::TimedOut
4430 {
4431 state.pending_terminal_override = Some(BgTaskStatus::TimedOut);
4432 }
4433
4434 #[cfg(windows)]
4435 let mut pty_forced_terminal_status: Option<BgTaskStatus> = None;
4436
4437 match &mut state.runtime {
4438 TaskRuntime::Piped(child_slot) => {
4439 #[cfg(unix)]
4440 if let Some(pgid) = pgid {
4441 terminate_pgid(pgid, child_slot.as_mut());
4442 }
4443 #[cfg(windows)]
4444 if let Some(child) = child_slot.as_mut() {
4445 super::process::terminate_process(child);
4446 } else if let Some(pid) = child_pid {
4447 terminate_pid(pid);
4448 }
4449 if let Some(child) = child_slot.as_mut() {
4450 let _ = child.wait();
4451 }
4452 *child_slot = None;
4453 state.detached = true;
4454
4455 if let Some(handles) = state.io_handles.as_mut() {
4456 match handles.write(TaskArtifact::Exit, b"killed") {
4457 Ok(()) => {}
4458 Err(error)
4459 if error.kind() == std::io::ErrorKind::Interrupted
4460 && error.to_string().contains(
4461 super::persistence::ARTIFACT_CONCURRENTLY_REPLACED,
4462 ) =>
4463 {
4464 write_kill_marker_if_absent(&task.paths).map_err(|e| {
4470 format!("failed to write kill marker after replace: {e}")
4471 })?;
4472 }
4473 Err(error) => {
4474 return Err(format!(
4475 "failed to write retained kill marker: {error}"
4476 ));
4477 }
4478 }
4479 } else {
4480 write_kill_marker_if_absent(&task.paths)
4481 .map_err(|e| format!("failed to write kill marker: {e}"))?;
4482 }
4483
4484 let exit_code = terminal_exit_code_for_status(&terminal_status);
4485 state
4486 .metadata
4487 .mark_terminal(terminal_status, exit_code, reason.clone());
4488
4489 state.pending_terminal_override = None;
4490 task.mark_terminal_now();
4491 self.persist_task(&task.paths, &state.metadata)
4492 .map_err(|e| format!("failed to persist killed state: {e}"))?;
4493 terminalized = true;
4494 }
4495 TaskRuntime::Pty(Some(pty)) => {
4496 pty.was_killed.store(true, Ordering::SeqCst);
4497 if let Err(error) = pty.killer.kill() {
4498 crate::slog_warn!(
4499 "[pty-kill] {task_id} ChildKiller::kill failed: {error}"
4500 );
4501 }
4502 if let Some(pid) = pty.child_pid {
4503 #[cfg(unix)]
4504 terminate_pgid(pid as i32, None);
4505 #[cfg(windows)]
4506 terminate_pid(pid);
4507 }
4508 drop(pty.master.take());
4509
4510 #[cfg(windows)]
4511 {
4512 let default_status = if terminal_status == BgTaskStatus::TimedOut {
4513 BgTaskStatus::TimedOut
4514 } else {
4515 BgTaskStatus::Killed
4516 };
4517 pty_forced_terminal_status = Some(
4518 state
4519 .pending_terminal_override
4520 .take()
4521 .unwrap_or(default_status),
4522 );
4523 }
4524 }
4525 TaskRuntime::Pty(None) => {}
4526 }
4527
4528 #[cfg(windows)]
4529 if let Some(target_status) = pty_forced_terminal_status {
4530 if !task.paths.exit.exists() {
4531 write_kill_marker_if_absent(&task.paths)
4532 .map_err(|e| format!("failed to write kill marker: {e}"))?;
4533 }
4534
4535 let exit_code = terminal_exit_code_for_status(&target_status);
4536 state
4537 .metadata
4538 .mark_terminal(target_status, exit_code, reason.clone());
4539
4540 state.pending_terminal_override = None;
4541 task.mark_terminal_now();
4542 if let TaskRuntime::Pty(runtime) = &mut state.runtime {
4543 *runtime = None;
4544 }
4545 state.detached = true;
4546 self.persist_task(&task.paths, &state.metadata)
4547 .map_err(|e| format!("failed to persist killed PTY state: {e}"))?;
4548 terminalized = true;
4549 }
4550 }
4551 }
4552
4553 if terminalized {
4554 self.post_terminal_transition(&task, true)?;
4555 }
4556 let mut snapshot = self.snapshot_with_terminal_cache(&task, RUNNING_OUTPUT_PREVIEW_BYTES);
4557 snapshot.kill_signaled = kill_signaled;
4558 snapshot.kill_reached = kill_reached;
4559 Ok(snapshot)
4560 }
4561
4562 fn finalize_from_marker(
4563 &self,
4564 task: &Arc<BgTask>,
4565 marker: ExitMarker,
4566 reason: Option<String>,
4567 ) -> Result<(), String> {
4568 let mut pty_reader_done = None;
4569 {
4570 let mut state = task
4571 .state
4572 .lock()
4573 .map_err(|_| "background task lock poisoned".to_string())?;
4574 if state.metadata.status.is_terminal() {
4575 state.pending_terminal_override = None;
4576 return Ok(());
4577 }
4578
4579 let pending_override = state.pending_terminal_override.take();
4580 let is_pty = state.metadata.mode == BgMode::Pty;
4581 let reason = reason.or_else(|| state.metadata.status_reason.clone());
4582 let updated = self
4583 .update_task_metadata(&task.paths, |metadata| {
4584 let new_metadata = if is_pty && marker == ExitMarker::Killed {
4585 let mut metadata = metadata.clone();
4586 let target_status = pending_override.unwrap_or(BgTaskStatus::Killed);
4587 let exit_code = terminal_exit_code_for_status(&target_status);
4588 metadata.mark_terminal(target_status, exit_code, reason.clone());
4589 metadata
4590 } else {
4591 terminal_metadata_from_marker(metadata.clone(), marker, reason.clone())
4592 };
4593 *metadata = new_metadata;
4594 })
4595 .map_err(|e| format!("failed to persist terminal state: {e}"))?;
4596 state.metadata = updated;
4597 task.mark_terminal_now();
4598 match &mut state.runtime {
4599 TaskRuntime::Piped(child_slot) => reap_piped_child(child_slot),
4604 TaskRuntime::Pty(runtime) => {
4605 pty_reader_done = runtime
4606 .as_ref()
4607 .map(|runtime| Arc::clone(&runtime.reader_done));
4608 *runtime = None;
4609 }
4610 }
4611 state.detached = true;
4612 }
4613
4614 if let Some(reader_done) = pty_reader_done {
4615 let deadline = Instant::now() + Duration::from_millis(200);
4616 while !reader_done.load(Ordering::SeqCst) && Instant::now() < deadline {
4617 std::thread::sleep(Duration::from_millis(10));
4618 }
4619 }
4620
4621 self.scan_task_watch_output(task);
4624
4625 self.post_terminal_transition(task, true)
4626 }
4627
4628 fn enqueue_completion_if_needed(
4629 &self,
4630 metadata: &PersistedTask,
4631 paths: Option<&TaskPaths>,
4632 emit_frame: bool,
4633 ) {
4634 if metadata.status.is_terminal() && !metadata.completion_delivered {
4635 let cache =
4636 paths.and_then(|paths| self.render_terminal_output_from_paths(metadata, paths));
4637 self.enqueue_completion_from_parts(metadata, None, paths, emit_frame, cache.as_ref());
4638 }
4639 }
4640
4641 fn render_terminal_output_from_paths(
4642 &self,
4643 metadata: &PersistedTask,
4644 paths: &TaskPaths,
4645 ) -> Option<TerminalOutputCache> {
4646 if metadata.mode == BgMode::Pty {
4647 return None;
4648 }
4649 let mut buffer = BgBuffer::registered(paths, BgMode::Pipes);
4650 let disk_truncation = buffer.enforce_terminal_cap();
4651 Some(self.render_terminal_output(metadata, &buffer, disk_truncation, Some(paths)))
4652 }
4653
4654 fn enqueue_completion_from_parts(
4655 &self,
4656 metadata: &PersistedTask,
4657 buffer: Option<&BgBuffer>,
4658 paths: Option<&TaskPaths>,
4659 emit_frame: bool,
4660 terminal_render: Option<&TerminalOutputCache>,
4661 ) {
4662 if !metadata.status.is_terminal() {
4673 return;
4674 }
4675
4676 let owned_buffer = if buffer.is_none() && metadata.mode != BgMode::Pty {
4677 paths.map(|paths| BgBuffer::registered(paths, BgMode::Pipes))
4678 } else {
4679 None
4680 };
4681 let render_buffer = buffer.or(owned_buffer.as_ref());
4682 let owned_render = if terminal_render.is_none() {
4683 render_buffer.map(|buffer| {
4684 let mut capped_buffer = buffer.clone();
4685 let disk_truncation = capped_buffer.enforce_terminal_cap();
4686 self.render_terminal_output(metadata, &capped_buffer, disk_truncation, paths)
4687 })
4688 } else {
4689 None
4690 };
4691 let render = terminal_render.or(owned_render.as_ref());
4692
4693 let (mut output_preview, output_truncated) = render
4697 .map(|cache| completion_preview_for_cache(cache, metadata.exit_code))
4698 .unwrap_or_else(|| (String::new(), false));
4699 if metadata.status == BgTaskStatus::FateUnknown {
4700 if let Some(reason) = metadata.status_reason.as_deref() {
4701 output_preview = if output_preview.is_empty() {
4702 reason.to_string()
4703 } else {
4704 format!("{reason}\n{output_preview}")
4705 };
4706 }
4707 }
4708 let bash_output_list_envelope =
4709 render.and_then(|cache| append_bash_output_envelope(cache, &mut output_preview));
4710
4711 let token_counts = self.completion_token_counts(
4712 metadata,
4713 buffer,
4714 paths,
4715 render.map(|render| render.output_preview.as_str()),
4716 );
4717 let completion = BgCompletion {
4718 task_id: metadata.task_id.clone(),
4719 session_id: metadata.session_id.clone(),
4720 status: metadata.status.clone(),
4721 exit_code: metadata.exit_code,
4722 command: metadata.command.clone(),
4723 output_preview,
4724 bash_output_list_envelope,
4725 output_truncated,
4726 original_tokens: token_counts.original_tokens,
4727 compressed_tokens: token_counts.compressed_tokens,
4728 tokens_skipped: token_counts.tokens_skipped,
4729 status_reason: metadata.status_reason.clone(),
4730 live_descendants: metadata.live_descendants.clone(),
4731 live_descendants_omitted: metadata.live_descendants_omitted,
4732 live_descendants_summary: live_descendants_summary(metadata),
4733 };
4734
4735 self.record_compression_event_if_applicable(metadata, &token_counts);
4746
4747 self.sync_memory_watches_from_persistence(&metadata.task_id);
4750 let (watch_controlled, watch_matched) = self.task_watch_state(&metadata.task_id);
4751 if watch_controlled {
4752 if !watch_matched && !metadata.completion_delivered {
4753 self.record_bash_watch_exit(&completion, emit_frame);
4754 }
4755 self.clear_task_watch_state(&metadata.task_id);
4758 return;
4759 }
4760
4761 if metadata.completion_delivered {
4771 return;
4772 }
4773
4774 let pushed = if let Ok(mut completions) = self.inner.completions.lock() {
4777 if completions
4778 .iter()
4779 .any(|existing| existing.task_id == metadata.task_id)
4780 {
4781 false
4782 } else {
4783 completions.push_back(completion.clone());
4784 true
4785 }
4786 } else {
4787 false
4788 };
4789
4790 if pushed && emit_frame {
4791 self.emit_bash_completed(completion);
4792 }
4793 }
4794
4795 fn record_compression_event_if_applicable(
4796 &self,
4797 metadata: &PersistedTask,
4798 token_counts: &CompletionTokenCounts,
4799 ) {
4800 if metadata.mode == BgMode::Pty {
4801 return;
4802 }
4803
4804 let (original_tokens, compressed_tokens, original_bytes, compressed_bytes) = match (
4805 token_counts.original_tokens,
4806 token_counts.compressed_tokens,
4807 token_counts.original_bytes,
4808 token_counts.compressed_bytes,
4809 ) {
4810 (
4811 Some(original_tokens),
4812 Some(compressed_tokens),
4813 Some(original_bytes),
4814 Some(compressed_bytes),
4815 ) => (
4816 original_tokens,
4817 compressed_tokens,
4818 original_bytes,
4819 compressed_bytes,
4820 ),
4821 _ => {
4822 crate::slog_warn!(
4823 "compression event skipped for {}: token counts unavailable (likely spill file missing or unreadable)",
4824 metadata.task_id
4825 );
4826 return;
4827 }
4828 };
4829
4830 let pool = self.inner.db_pool.read().ok().and_then(|slot| slot.clone());
4831 let Some(pool) = pool else {
4832 crate::slog_warn!(
4833 "compression event skipped for {}: db_pool not initialized — was configure run?",
4834 metadata.task_id
4835 );
4836 return;
4837 };
4838 let harness = self
4839 .inner
4840 .db_harness
4841 .read()
4842 .ok()
4843 .and_then(|slot| slot.clone());
4844 let Some(harness) = harness else {
4845 crate::slog_warn!(
4846 "compression event insert skipped for {}: harness not configured",
4847 metadata.task_id
4848 );
4849 return;
4850 };
4851
4852 let project_root = metadata
4853 .project_root
4854 .as_deref()
4855 .unwrap_or(&metadata.workdir);
4856 let project_key = crate::path_identity::project_scope_key(project_root);
4857 let row = crate::db::compression_events::CompressionEventRow {
4858 harness: &harness,
4859 session_id: Some(&metadata.session_id),
4860 project_key: &project_key,
4861 tool: "bash",
4862 task_id: Some(&metadata.task_id),
4863 command: Some(&metadata.command),
4864 compressor: if metadata.compressed {
4865 "registry"
4866 } else {
4867 "none"
4868 },
4869 original_bytes,
4870 compressed_bytes,
4871 original_tokens,
4872 compressed_tokens,
4873 created_at: unix_millis() as i64,
4874 };
4875
4876 let conn = match pool.lock() {
4877 Ok(conn) => conn,
4878 Err(_) => {
4879 crate::slog_warn!(
4880 "compression event insert failed for {}: db mutex poisoned",
4881 metadata.task_id
4882 );
4883 return;
4884 }
4885 };
4886 match crate::db::compression_events::insert_compression_event(&conn, &row) {
4887 Ok(Some(row_id)) => {
4888 self.inner
4892 .compression_aggregates
4893 .record_successful_insert(&conn, &row, row_id);
4894 crate::slog_debug!(
4898 "compression event recorded for {} (project={}, session={}, {} → {} tokens)",
4899 metadata.task_id,
4900 project_key,
4901 metadata.session_id,
4902 original_tokens,
4903 compressed_tokens
4904 );
4905 }
4906 Ok(None) => {
4907 crate::slog_debug!(
4908 "duplicate compression event ignored for {} (project={}, session={})",
4909 metadata.task_id,
4910 project_key,
4911 metadata.session_id
4912 );
4913 }
4914 Err(error) => {
4915 crate::slog_warn!(
4916 "compression event insert failed for {}: {}",
4917 metadata.task_id,
4918 error
4919 );
4920 }
4921 }
4922 }
4923
4924 fn emit_bash_pattern_match(&self, session_id: &str, pattern_match: PatternMatch) {
4925 if !self.originating_session_has_live_route(session_id) {
4926 return;
4927 }
4928 let Ok(progress_sender) = self
4929 .inner
4930 .progress_sender
4931 .lock()
4932 .map(|sender| sender.clone())
4933 else {
4934 return;
4935 };
4936 if let Some(sender) = progress_sender.as_ref() {
4937 let frame = if pattern_match.match_text == WATCH_TASK_EXIT_TEXT {
4938 BashPatternMatchFrame::task_exit(
4939 pattern_match.task_id,
4940 session_id.to_string(),
4941 pattern_match.match_text,
4942 pattern_match.context,
4943 )
4944 } else {
4945 BashPatternMatchFrame::new(
4946 pattern_match.task_id,
4947 session_id.to_string(),
4948 pattern_match.watch_id,
4949 pattern_match.match_text,
4950 pattern_match.match_offset,
4951 pattern_match.context,
4952 pattern_match.once,
4953 )
4954 };
4955 sender(PushFrame::BashPatternMatch(frame));
4956 }
4957 }
4958
4959 fn emit_bash_watch_erased(&self, session_id: &str, task_id: &str, watch_id: &str) {
4960 if !self.originating_session_has_live_route(session_id) {
4961 return;
4962 }
4963 let Ok(progress_sender) = self
4964 .inner
4965 .progress_sender
4966 .lock()
4967 .map(|sender| sender.clone())
4968 else {
4969 return;
4970 };
4971 let Some(sender) = progress_sender.as_ref() else {
4972 return;
4973 };
4974 sender(PushFrame::BashPatternMatch(
4975 BashPatternMatchFrame::watch_target_erased(
4976 task_id,
4977 session_id,
4978 watch_id,
4979 WATCH_TARGET_ERASED_TEXT,
4980 WATCH_TARGET_ERASED_CONTEXT,
4981 ),
4982 ));
4983 }
4984
4985 fn record_bash_watch_exit(&self, completion: &BgCompletion, emit_frame: bool) {
4986 let status = completion_status_text(&completion.status, completion.exit_code);
4987 let preview = completion.output_preview.trim_end();
4988 let context = if preview.is_empty() {
4989 format!("task {} exited ({status})", completion.task_id)
4990 } else {
4991 format!(
4992 "task {} exited ({status})
4993{preview}",
4994 completion.task_id
4995 )
4996 };
4997 let frame = BashPatternMatchFrame::task_exit(
4998 completion.task_id.clone(),
4999 completion.session_id.clone(),
5000 format!("exited ({status})"),
5001 context,
5002 );
5003 self.persist_bash_watch_exit(&frame);
5004 if emit_frame {
5005 self.emit_bash_watch_exit(frame);
5006 }
5007 }
5008
5009 fn persist_bash_watch_exit(&self, frame: &BashPatternMatchFrame) {
5010 let Some((harness, pool)) = self.db_harness_and_pool() else {
5011 return;
5012 };
5013 let Ok(conn) = pool.lock() else {
5014 return;
5015 };
5016 let Ok(mut rows) = crate::db::bash_watches::list_bash_pattern_watches_for_task(
5017 &conn,
5018 &harness,
5019 &frame.session_id,
5020 &frame.task_id,
5021 ) else {
5022 return;
5023 };
5024 let Some(mut row) = rows
5025 .iter()
5026 .position(|row| row.match_text.as_deref() == Some(WATCH_TASK_EXIT_TEXT))
5027 .map(|index| rows.swap_remove(index))
5028 .or_else(|| rows.into_iter().next())
5029 else {
5030 return;
5031 };
5032 row.scanning = false;
5033 row.pending_match = true;
5034 row.match_text = Some(WATCH_TASK_EXIT_TEXT.to_string());
5035 row.match_offset = Some(0);
5036 row.match_context = Some(frame.context.clone());
5037 if let Err(error) = crate::db::bash_watches::upsert_bash_pattern_watch(&conn, &row) {
5038 crate::slog_warn!(
5039 "persist bash watch task-exit failed for {}/{}: {error}",
5040 frame.task_id,
5041 row.watch_id
5042 );
5043 }
5044 }
5045
5046 fn emit_bash_watch_exit(&self, frame: BashPatternMatchFrame) {
5047 if !self.originating_session_has_live_route(&frame.session_id) {
5048 return;
5049 }
5050 let Ok(progress_sender) = self
5051 .inner
5052 .progress_sender
5053 .lock()
5054 .map(|sender| sender.clone())
5055 else {
5056 return;
5057 };
5058 let Some(sender) = progress_sender.as_ref() else {
5059 return;
5060 };
5061 sender(PushFrame::BashPatternMatch(frame));
5062 }
5063
5064 fn emit_bash_completed(&self, completion: BgCompletion) {
5065 if !self.originating_session_has_live_route(&completion.session_id) {
5066 return;
5067 }
5068 let Ok(progress_sender) = self
5069 .inner
5070 .progress_sender
5071 .lock()
5072 .map(|sender| sender.clone())
5073 else {
5074 return;
5075 };
5076 let Some(sender) = progress_sender.as_ref() else {
5077 return;
5078 };
5079 let mut frame = BashCompletedFrame::new(
5087 completion.task_id,
5088 completion.session_id,
5089 completion.status,
5090 completion.exit_code,
5091 completion.command,
5092 completion.output_preview,
5093 completion.output_truncated,
5094 completion.original_tokens,
5095 completion.compressed_tokens,
5096 completion.tokens_skipped,
5097 );
5098 frame.bash_output_list_envelope = completion.bash_output_list_envelope;
5099 frame.status_reason = completion.status_reason;
5100 frame.live_descendants = completion.live_descendants;
5101 frame.live_descendants_omitted = completion.live_descendants_omitted;
5102 frame.live_descendants_summary = completion.live_descendants_summary;
5103 sender(PushFrame::BashCompleted(frame));
5104 }
5105
5106 fn completion_token_counts(
5107 &self,
5108 metadata: &PersistedTask,
5109 buffer: Option<&BgBuffer>,
5110 paths: Option<&TaskPaths>,
5111 rendered_output: Option<&str>,
5112 ) -> CompletionTokenCounts {
5113 if metadata.mode == BgMode::Pty {
5114 return CompletionTokenCounts::skipped();
5115 }
5116
5117 let raw = match buffer {
5118 Some(buffer) => buffer.read_for_token_count(TOKENIZE_CAP_BYTES_PER_STREAM),
5119 None => paths
5120 .map(|paths| {
5121 read_for_token_count_from_disk(metadata, paths, TOKENIZE_CAP_BYTES_PER_STREAM)
5122 })
5123 .unwrap_or(TokenCountInput::Skipped),
5124 };
5125
5126 let TokenCountInput::Text(raw_output) = raw else {
5127 return CompletionTokenCounts::skipped();
5128 };
5129
5130 let original_tokens = token_count_u32(&raw_output);
5131 let original_bytes = raw_output.len() as i64;
5132 let compressed_output = rendered_output.unwrap_or(&raw_output);
5133 let compressed_tokens = token_count_u32(compressed_output);
5134 let compressed_bytes = compressed_output.len() as i64;
5135 CompletionTokenCounts {
5136 original_tokens: Some(original_tokens),
5137 compressed_tokens: Some(compressed_tokens),
5138 original_bytes: Some(original_bytes),
5139 compressed_bytes: Some(compressed_bytes),
5140 tokens_skipped: false,
5141 }
5142 }
5143
5144 pub(crate) fn maybe_emit_long_running_reminder(&self, task: &Arc<BgTask>) {
5145 if !self
5146 .inner
5147 .long_running_reminder_enabled
5148 .load(Ordering::SeqCst)
5149 {
5150 return;
5151 }
5152 let interval_ms = self
5153 .inner
5154 .long_running_reminder_interval_ms
5155 .load(Ordering::SeqCst);
5156 if interval_ms == 0 {
5157 return;
5158 }
5159 let interval = Duration::from_millis(interval_ms);
5160 let now = Instant::now();
5161 let Ok(mut last_reminder_at) = task.last_reminder_at.lock() else {
5162 return;
5163 };
5164 let since = last_reminder_at.unwrap_or(task.started);
5165 if now.duration_since(since) < interval {
5166 return;
5167 }
5168 let command = task
5169 .state
5170 .lock()
5171 .map(|state| state.metadata.command.clone())
5172 .unwrap_or_default();
5173 *last_reminder_at = Some(now);
5174 self.emit_bash_long_running(BashLongRunningFrame::new(
5175 task.task_id.clone(),
5176 task.session_id.clone(),
5177 command,
5178 task.started.elapsed().as_millis() as u64,
5179 ));
5180 }
5181
5182 fn emit_bash_long_running(&self, frame: BashLongRunningFrame) {
5183 if !self.originating_session_has_live_route(&frame.session_id) {
5184 return;
5185 }
5186 let Ok(progress_sender) = self
5187 .inner
5188 .progress_sender
5189 .lock()
5190 .map(|sender| sender.clone())
5191 else {
5192 return;
5193 };
5194 if let Some(sender) = progress_sender.as_ref() {
5195 sender(PushFrame::BashLongRunning(frame));
5196 }
5197 }
5198
5199 fn task(&self, task_id: &str) -> Option<Arc<BgTask>> {
5200 validate_task_id(task_id).ok()?;
5201 self.inner
5202 .tasks
5203 .lock()
5204 .ok()
5205 .and_then(|tasks| tasks.get(task_id).cloned())
5206 }
5207
5208 fn task_for_session(&self, task_id: &str, session_id: &str) -> Option<Arc<BgTask>> {
5209 self.task(task_id)
5210 .filter(|task| task.session_id == session_id)
5211 }
5212
5213 pub fn try_health_counts(&self) -> Option<BgTaskHealthCounts> {
5214 let running = self
5215 .inner
5216 .tasks
5217 .try_lock()
5218 .ok()
5219 .map(|tasks| tasks.values().filter(|task| task.is_running()).count())?;
5220 let pending_completions = self.inner.completions.try_lock().ok().map(|q| q.len())?;
5221 Some(BgTaskHealthCounts {
5222 running,
5223 pending_completions,
5224 })
5225 }
5226
5227 pub(crate) fn detached_live_process_count(&self) -> usize {
5230 let Some(pids) = self.inner.tasks.try_lock().ok().map(|tasks| {
5231 tasks
5232 .values()
5233 .filter_map(|task| {
5234 task.state
5235 .try_lock()
5236 .ok()
5237 .and_then(|state| state.metadata.child_pid)
5238 })
5239 .collect::<Vec<_>>()
5240 }) else {
5241 return 0;
5242 };
5243 pids.into_iter()
5244 .filter(|pid| is_process_alive(*pid))
5245 .count()
5246 }
5247
5248 pub fn estimated_memory(&self) -> crate::memory::MemoryEstimate {
5252 let tasks = match self.inner.tasks.try_lock() {
5253 Ok(tasks) => tasks.values().cloned().collect::<Vec<_>>(),
5254 Err(_) => return crate::memory::MemoryEstimate::busy(),
5255 };
5256 let mut bytes = 0u64;
5257 let mut terminal_output_caches = 0usize;
5258 let mut sessions = HashSet::new();
5259 for task in &tasks {
5260 sessions.insert(task.session_id.clone());
5261 let state = match task.state.try_lock() {
5262 Ok(state) => state,
5263 Err(_) => return crate::memory::MemoryEstimate::busy(),
5264 };
5265 if let Some(cache) = state.terminal_output_cache.as_ref() {
5266 terminal_output_caches = terminal_output_caches.saturating_add(1);
5267 bytes = bytes.saturating_add(terminal_output_cache_estimated_bytes(cache));
5268 }
5269 }
5270 let completion_count = match self.inner.completions.try_lock() {
5271 Ok(completions) => {
5272 for completion in completions.iter() {
5273 sessions.insert(completion.session_id.clone());
5274 bytes = bytes.saturating_add(completion_estimated_bytes(completion));
5275 }
5276 completions.len()
5277 }
5278 Err(_) => return crate::memory::MemoryEstimate::busy(),
5279 };
5280
5281 crate::memory::MemoryEstimate::estimated(bytes)
5282 .count("tasks", tasks.len())
5283 .count("sessions", sessions.len())
5284 .count("terminal_output_caches", terminal_output_caches)
5285 .count("completion_caches", completion_count)
5286 .count_u64("output_ring_bytes", 0)
5287 }
5288
5289 fn running_count(&self) -> usize {
5290 self.inner
5291 .tasks
5292 .lock()
5293 .map(|tasks| tasks.values().filter(|task| task.is_running()).count())
5294 .unwrap_or(0)
5295 }
5296
5297 fn start_watchdog(&self) {
5298 if !self.inner.watchdog_started.swap(true, Ordering::SeqCst) {
5299 super::watchdog::start(self.clone());
5300 }
5301 }
5302
5303 #[cfg(test)]
5304 pub fn task_json_path(&self, task_id: &str, session_id: &str) -> Option<PathBuf> {
5305 self.task_for_session(task_id, session_id)
5306 .map(|task| task.paths.json.clone())
5307 }
5308
5309 #[cfg(test)]
5310 pub fn task_exit_path(&self, task_id: &str, session_id: &str) -> Option<PathBuf> {
5311 self.task_for_session(task_id, session_id)
5312 .map(|task| task.paths.exit.clone())
5313 }
5314}
5315
5316#[cfg(unix)]
5317fn should_capture_pipeline_status(
5318 spawn_plan: &SpawnPlan,
5319 has_pipeline: bool,
5320 shell: &Path,
5321) -> bool {
5322 if spawn_plan.is_native_launcher() {
5323 return false;
5326 }
5327 has_pipeline && super::process::pipeline_shell_kind(shell).is_some()
5328}
5329
5330fn canonical_artifact_root(paths: &TaskPaths) -> PathBuf {
5331 fs::canonicalize(&paths.io_dir).unwrap_or_else(|_| paths.io_dir.clone())
5332}
5333
5334fn restart_fate_unknown_reason(metadata: &PersistedTask, paths: &TaskPaths) -> String {
5335 let output = match metadata.mode {
5336 BgMode::Pipes => &paths.stdout,
5337 BgMode::Pty => &paths.pty,
5338 };
5339 format!(
5340 "task {}: daemon restarted, process fate unknown, last output at {}",
5341 metadata.task_id,
5342 output.display()
5343 )
5344}
5345
5346fn append_pipeline_warning(
5351 cache: &mut TerminalOutputCache,
5352 metadata: &PersistedTask,
5353 paths: Option<&TaskPaths>,
5354) {
5355 if metadata.exit_code != Some(0) {
5356 return;
5357 }
5358 let Some(paths) = paths else {
5359 return;
5360 };
5361 if metadata.pipeline_segments.len() < 2 {
5362 return;
5363 }
5364 let mut status_file = match open_task_artifact(paths, TaskArtifact::PipelineStatus) {
5365 Ok(file) => file,
5366 Err(_) => {
5367 let Some(shell) = metadata.pipeline_status_unavailable.as_deref() else {
5368 return;
5369 };
5370 let footer = format!(
5371 "note: pipeline status unavailable under {shell}; an upstream failure may be masked by the final segment's exit code."
5372 );
5373 if cache.output_preview.trim().is_empty() {
5374 cache.output_preview = footer;
5375 } else {
5376 cache.output_preview = format!("{}\n{footer}", cache.output_preview.trim_end());
5377 }
5378 return;
5379 }
5380 };
5381 let Ok(status_bytes) = status_file.read_all() else {
5382 return;
5383 };
5384 let Some(statuses) = String::from_utf8_lossy(&status_bytes)
5385 .lines()
5386 .map(|line| line.trim().parse::<i32>().ok())
5387 .collect::<Option<Vec<_>>>()
5388 else {
5389 return;
5390 };
5391 if statuses.len() != metadata.pipeline_segments.len() {
5392 return;
5393 }
5394 let Some((failing_index, failing_code)) = statuses
5395 .iter()
5396 .enumerate()
5397 .take(statuses.len().saturating_sub(1))
5398 .find(|(_, code)| **code != 0)
5399 .map(|(index, code)| (index, *code))
5400 else {
5401 return;
5402 };
5403 let Some(final_segment) = metadata.pipeline_segments.last() else {
5404 return;
5405 };
5406 let failing_segment = &metadata.pipeline_segments[failing_index];
5407 let footer = format!(
5408 "note: `{}` (segment {} of {}) exited {}; the pipeline's exit code is `{}`'s.",
5409 failing_segment,
5410 failing_index + 1,
5411 metadata.pipeline_segments.len(),
5412 failing_code,
5413 final_segment,
5414 );
5415 if cache.output_preview.trim().is_empty() {
5416 cache.output_preview = footer;
5417 } else {
5418 cache.output_preview = format!("{}\n{}", cache.output_preview.trim_end(), footer,);
5419 }
5420}
5421
5422fn normalize_piped_display_output(text: &mut String) {
5425 if !text.contains('\r') {
5426 return;
5427 }
5428
5429 let mut rendered = String::with_capacity(text.len());
5430 let mut line = Vec::new();
5431 let mut column = 0;
5432 let mut chars = text.chars().peekable();
5433
5434 while let Some(character) = chars.next() {
5435 match character {
5436 '\r' if chars.peek() == Some(&'\n') => {
5437 chars.next();
5438 for character in &line {
5439 rendered.push(*character);
5440 }
5441 rendered.push('\n');
5442 line.clear();
5443 column = 0;
5444 }
5445 '\r' => column = 0,
5446 '\n' => {
5447 for character in &line {
5448 rendered.push(*character);
5449 }
5450 rendered.push('\n');
5451 line.clear();
5452 column = 0;
5453 }
5454 character => {
5455 if column < line.len() {
5456 line[column] = character;
5457 } else {
5458 line.resize(column, ' ');
5459 line.push(character);
5460 }
5461 column += 1;
5462 }
5463 }
5464 }
5465
5466 for character in &line {
5467 rendered.push(*character);
5468 }
5469 *text = rendered;
5470}
5471
5472fn render_compressed_with_recovery(
5473 buffer: &BgBuffer,
5474 mut compressed: CompressionResult,
5475 input_truncated: bool,
5476 disk_truncation: DiskTruncation,
5477 artifact_access: ArtifactRecoveryAccess,
5478) -> TerminalOutputCache {
5479 let compression_input_line_count = compressed.input_line_count;
5487 let had_trailing_newline = compressed.text.ends_with('\n');
5488 let mut text = strip_plain_truncation_marker_lines(&compressed.text)
5489 .trim_end()
5490 .to_string();
5491 if had_trailing_newline && !text.is_empty() {
5492 text.push('\n');
5493 }
5494 compressed.text = text;
5495
5496 let output_path = buffer.output_path().map(|path| path.display().to_string());
5497 let stderr_path = buffer.stderr_path().map(|path| path.display().to_string());
5498 let include_stderr_path = buffer.stream_len(StreamKind::Stderr) > 0;
5499 let mut recovery = RecoveryContext {
5500 dropped_by_class: compressed.dropped_by_class,
5501 had_inner_drop: compressed.had_inner_drop,
5502 offset_hint_eligible: compressed.offset_hint_eligible,
5503 offset_start_line: compressed.offset_start_line,
5504 byte_truncated: input_truncated,
5505 disk_truncated_prefix_bytes: disk_truncation.total_prefix_bytes(),
5506 output_path: output_path.clone(),
5507 stderr_path: stderr_path.clone(),
5508 include_stderr_path,
5509 artifact_access: artifact_access.clone(),
5510 };
5511
5512 let (output_preview, output_truncated) =
5513 render_body_with_recovery_marker(&compressed.text, &mut recovery);
5514 TerminalOutputCache {
5515 output_preview,
5516 output_truncated,
5517 compression_input_line_count: Some(compression_input_line_count),
5518 kind: TerminalOutputKind::Compressed,
5519 output_path,
5520 stderr_path,
5521 artifact_access,
5522 recovery: Some(recovery),
5523 }
5524}
5525
5526fn render_body_with_recovery_marker(body: &str, recovery: &mut RecoveryContext) -> (String, bool) {
5527 render_body_with_recovery_marker_at_cap(
5528 body,
5529 recovery,
5530 FINAL_OUTPUT_CAP_BYTES,
5531 cap_final_output,
5532 cap_final_output_with_marker,
5533 )
5534}
5535
5536fn render_raw_body_with_recovery_marker(
5537 body: &str,
5538 recovery: &mut RecoveryContext,
5539) -> (String, bool) {
5540 render_body_with_recovery_marker_at_cap(
5541 body,
5542 recovery,
5543 RAW_PASSTHROUGH_CAP_BYTES,
5544 |input| {
5545 super::output::cap_head_tail(
5546 input,
5547 RAW_PASSTHROUGH_CAP_BYTES,
5548 RAW_PASSTHROUGH_HEAD_BYTES,
5549 RAW_PASSTHROUGH_TAIL_BYTES,
5550 )
5551 },
5552 |input, marker| {
5553 super::output::cap_head_tail_with_marker(
5554 input,
5555 RAW_PASSTHROUGH_CAP_BYTES,
5556 RAW_PASSTHROUGH_HEAD_BYTES,
5557 RAW_PASSTHROUGH_TAIL_BYTES,
5558 marker,
5559 )
5560 },
5561 )
5562}
5563
5564fn render_body_with_recovery_marker_at_cap<F, G>(
5565 body: &str,
5566 recovery: &mut RecoveryContext,
5567 cap_bytes: usize,
5568 cap_plain: F,
5569 cap_with_marker: G,
5570) -> (String, bool)
5571where
5572 F: Fn(&str) -> super::output::CappedText,
5573 G: Fn(&str, &str) -> super::output::CappedText,
5574{
5575 let needs_marker = recovery.has_visible_drop();
5576 if body.len() > cap_bytes {
5577 recovery.byte_truncated = true;
5578 if let Some(marker) = recovery_marker(recovery) {
5579 let capped = cap_with_marker(body, &marker);
5580 return (capped.text, true);
5581 }
5582 let capped = cap_plain(body);
5583 return (capped.text, capped.truncated || needs_marker);
5584 }
5585
5586 if !needs_marker {
5587 return (body.to_string(), false);
5588 }
5589
5590 let Some(marker) = recovery_marker(recovery) else {
5591 return (body.to_string(), true);
5592 };
5593 let with_marker = append_recovery_marker(body, &marker);
5594 if with_marker.len() <= cap_bytes {
5595 return (with_marker, true);
5596 }
5597
5598 recovery.byte_truncated = true;
5599 let marker = recovery_marker(recovery).unwrap_or(marker);
5600 let capped = cap_with_marker(body, &marker);
5601 (capped.text, true)
5602}
5603
5604fn append_recovery_marker(body: &str, marker: &str) -> String {
5605 if body.is_empty() {
5606 return marker.to_string();
5607 }
5608 let mut output = body.trim_end().to_string();
5609 output.push('\n');
5610 output.push_str(marker);
5611 output
5612}
5613
5614fn recovery_marker(recovery: &RecoveryContext) -> Option<String> {
5615 let mut parts = Vec::new();
5616 for (class, count) in &recovery.dropped_by_class {
5617 let label = if *count == 1 {
5618 class.singular()
5619 } else {
5620 class.plural()
5621 };
5622 parts.push(format!("+{count} more {label}"));
5623 }
5624 if recovery.byte_truncated {
5625 parts.push("truncated output".to_string());
5626 }
5627 let disk_truncated_prefix_bytes = recovery.disk_truncated_prefix_bytes;
5628 if disk_truncated_prefix_bytes > 0 {
5629 parts.push(format!(
5630 "truncated {disk_truncated_prefix_bytes} bytes from saved output prefix"
5631 ));
5632 } else if recovery.had_inner_drop && parts.is_empty() {
5633 parts.push("omitted output".to_string());
5634 }
5635
5636 if parts.is_empty() {
5637 return None;
5638 }
5639
5640 let hint = recovery_hint(recovery);
5641 Some(format!("[{}; {hint}]", parts.join(", ")))
5642}
5643
5644fn bash_status_recovery_hint(access: &ArtifactRecoveryAccess) -> String {
5645 let task_id = serde_json::to_string(&access.task_id)
5646 .unwrap_or_else(|_| format!("\"{}\"", access.task_id));
5647 format!("use bash_status({{taskId: {task_id}}})")
5648}
5649
5650fn recovery_hint(recovery: &RecoveryContext) -> String {
5651 if !recovery.artifact_access.readable {
5652 return bash_status_recovery_hint(&recovery.artifact_access);
5653 }
5654
5655 if recovery.offset_hint_eligible
5659 && !recovery.byte_truncated
5660 && recovery.dropped_by_class.is_empty()
5661 && !recovery.include_stderr_path
5662 {
5663 if let (Some(path), Some(line)) =
5664 (recovery.output_path.as_deref(), recovery.offset_start_line)
5665 {
5666 return format!("see remaining: tail -n +{line} {}", quote_path(path));
5667 }
5668 }
5669
5670 let mut paths = Vec::new();
5671 if let Some(path) = recovery.output_path.as_deref() {
5672 paths.push(path);
5673 }
5674 if recovery.include_stderr_path {
5675 if let Some(path) = recovery.stderr_path.as_deref() {
5676 if !paths.contains(&path) {
5677 paths.push(path);
5678 }
5679 }
5680 }
5681
5682 if paths.is_empty() {
5683 return "full output unavailable".to_string();
5684 }
5685
5686 let reads = paths
5687 .into_iter()
5688 .map(|path| format!("read {}", quote_path(path)))
5689 .collect::<Vec<_>>()
5690 .join(" and ");
5691 if recovery.disk_truncated_prefix_bytes > 0 {
5692 format!("retained output: {reads}")
5693 } else {
5694 format!("full output: {reads}")
5695 }
5696}
5697
5698fn strip_plain_truncation_marker_lines(input: &str) -> String {
5699 input
5700 .lines()
5701 .filter(|line| !is_plain_truncation_marker(line.trim()))
5702 .collect::<Vec<_>>()
5703 .join("\n")
5704}
5705
5706fn strip_recovery_marker_lines(input: &str) -> String {
5707 input
5708 .lines()
5709 .filter(|line| !is_recovery_marker(line.trim()))
5710 .collect::<Vec<_>>()
5711 .join("\n")
5712}
5713
5714fn is_plain_truncation_marker(line: &str) -> bool {
5715 let Some(rest) = line.strip_prefix("...<truncated ") else {
5716 return false;
5717 };
5718 let Some(bytes) = rest.strip_suffix(" bytes>...") else {
5719 return false;
5720 };
5721 !bytes.is_empty() && bytes.chars().all(|ch| ch.is_ascii_digit())
5722}
5723
5724fn is_recovery_marker(line: &str) -> bool {
5725 line.starts_with('[')
5726 && line.ends_with(']')
5727 && (line.contains("full output: read ")
5728 || line.contains("retained output: read ")
5729 || line.contains("see remaining: tail -n +")
5730 || line.contains("use bash_status({taskId:")
5731 || line.contains("full output unavailable"))
5732}
5733
5734fn structured_output_pointer(
5735 total_bytes: u64,
5736 output_path: &str,
5737 truncated_prefix_bytes: u64,
5738 artifact_access: &ArtifactRecoveryAccess,
5739) -> String {
5740 if artifact_access.readable {
5741 return if truncated_prefix_bytes > 0 {
5742 retained_json_output_pointer(total_bytes, output_path, truncated_prefix_bytes)
5743 } else {
5744 json_output_pointer(total_bytes, output_path)
5745 };
5746 }
5747
5748 let kb = total_bytes.div_ceil(1024);
5749 let hint = bash_status_recovery_hint(artifact_access);
5750 if truncated_prefix_bytes > 0 {
5751 format!(
5752 "[JSON output {kb} KB; truncated {truncated_prefix_bytes} bytes from saved output prefix; retained output: {hint}]"
5753 )
5754 } else {
5755 format!("[JSON output {kb} KB; full output: {hint}]")
5756 }
5757}
5758
5759fn render_structured_output(
5760 command: &str,
5761 buffer: &BgBuffer,
5762 disk_truncation: DiskTruncation,
5763 artifact_access: ArtifactRecoveryAccess,
5764) -> Option<TerminalOutputCache> {
5765 if !is_gh_structured_command(command) {
5766 return None;
5767 }
5768
5769 let output_path = buffer
5770 .output_path()
5771 .map(|path| path.display().to_string())?;
5772 let stdout_bytes = buffer.stream_len(StreamKind::Stdout);
5773 if stdout_bytes == 0 {
5774 return None;
5775 }
5776
5777 if stdout_bytes > STRUCTURED_OUTPUT_CAP_BYTES as u64 {
5778 if !stream_starts_like_json(buffer, StreamKind::Stdout) {
5779 return None;
5780 }
5781 let output_preview = structured_output_pointer(
5782 stdout_bytes,
5783 &output_path,
5784 disk_truncation.total_prefix_bytes(),
5785 &artifact_access,
5786 );
5787 return Some(TerminalOutputCache {
5788 output_preview,
5789 output_truncated: true,
5790 compression_input_line_count: None,
5791 kind: TerminalOutputKind::Structured,
5792 output_path: Some(output_path),
5793 stderr_path: buffer.stderr_path().map(|path| path.display().to_string()),
5794 artifact_access,
5795 recovery: None,
5796 });
5797 }
5798
5799 let stdout = buffer.read_stream_bounded(StreamKind::Stdout, STRUCTURED_OUTPUT_CAP_BYTES);
5800 if stdout.truncated || !is_structured_body(&stdout.text) {
5801 return None;
5802 }
5803
5804 Some(TerminalOutputCache {
5805 output_preview: stdout.text,
5806 output_truncated: false,
5807 compression_input_line_count: None,
5808 kind: TerminalOutputKind::Structured,
5809 output_path: Some(output_path),
5810 stderr_path: buffer.stderr_path().map(|path| path.display().to_string()),
5811 artifact_access,
5812 recovery: None,
5813 })
5814}
5815
5816fn render_raw_passthrough(
5817 buffer: &BgBuffer,
5818 disk_truncation: DiskTruncation,
5819 artifact_access: ArtifactRecoveryAccess,
5820) -> TerminalOutputCache {
5821 let raw = buffer.read_combined_head_tail(
5822 RAW_PASSTHROUGH_CAP_BYTES,
5823 RAW_PASSTHROUGH_HEAD_BYTES,
5824 RAW_PASSTHROUGH_TAIL_BYTES,
5825 );
5826 let output_path = buffer.output_path().map(|path| path.display().to_string());
5827 let stderr_path = buffer.stderr_path().map(|path| path.display().to_string());
5828 if !raw.truncated && disk_truncation.total_prefix_bytes() == 0 {
5829 return TerminalOutputCache {
5830 output_preview: raw.text,
5831 output_truncated: false,
5832 compression_input_line_count: None,
5833 kind: TerminalOutputKind::Raw,
5834 output_path,
5835 stderr_path,
5836 artifact_access,
5837 recovery: None,
5838 };
5839 }
5840
5841 let include_stderr_path = buffer.stream_len(StreamKind::Stderr) > 0;
5842 let mut recovery = RecoveryContext {
5843 dropped_by_class: BTreeMap::new(),
5844 had_inner_drop: false,
5845 offset_hint_eligible: false,
5846 offset_start_line: None,
5847 byte_truncated: raw.truncated,
5848 disk_truncated_prefix_bytes: disk_truncation.total_prefix_bytes(),
5849 output_path: output_path.clone(),
5850 stderr_path: stderr_path.clone(),
5851 include_stderr_path,
5852 artifact_access: artifact_access.clone(),
5853 };
5854 let (output_preview, output_truncated) =
5855 render_raw_body_with_recovery_marker(&raw.text, &mut recovery);
5856 TerminalOutputCache {
5857 output_preview,
5858 output_truncated,
5859 compression_input_line_count: None,
5860 kind: TerminalOutputKind::Raw,
5861 output_path,
5862 stderr_path,
5863 artifact_access,
5864 recovery: Some(recovery),
5865 }
5866}
5867
5868fn append_bash_output_envelope(
5869 cache: &TerminalOutputCache,
5870 output: &mut String,
5871) -> Option<ListEnvelope> {
5872 cache
5873 .compression_input_line_count
5874 .and_then(|total| crate::list_surfaces::bash::append_envelope_trailer(output, total))
5875}
5876
5877fn completion_preview_for_cache(
5878 cache: &TerminalOutputCache,
5879 exit_code: Option<i32>,
5880) -> (String, bool) {
5881 let exit_ok = exit_code == Some(0);
5884 let threshold = completion_preview_threshold(exit_ok);
5885 if cache.kind == TerminalOutputKind::Structured && cache.output_preview.len() > threshold {
5886 if let Some(path) = cache.output_path.as_deref() {
5887 return (
5888 structured_output_pointer(
5889 cache.output_preview.len() as u64,
5890 path,
5891 0,
5892 &cache.artifact_access,
5893 ),
5894 true,
5895 );
5896 }
5897 return (cache.output_preview.clone(), cache.output_truncated);
5898 }
5899
5900 if let Some(recovery) = cache.recovery.as_ref() {
5901 if cache.output_preview.len() <= threshold {
5902 return (cache.output_preview.clone(), cache.output_truncated);
5903 }
5904 let body = strip_recovery_marker_lines(&cache.output_preview);
5905 let mut completion_recovery = recovery.clone();
5906 completion_recovery.byte_truncated = true;
5907 if let Some(marker) = recovery_marker(&completion_recovery) {
5908 let capped = cap_completion_output_with_marker(&body, &marker, exit_ok);
5909 return (capped.text, true);
5910 }
5911 }
5912
5913 let capped = cap_completion_output(&cache.output_preview, exit_ok);
5914 (capped.text, cache.output_truncated || capped.truncated)
5915}
5916
5917fn is_gh_structured_command(command: &str) -> bool {
5918 let Some(normalized) = crate::compress::plain_command_for_structured_output(command) else {
5919 return false;
5920 };
5921 let tokens = shell_words_for_flags(&normalized);
5922 let Some(head) = tokens.first() else {
5923 return false;
5924 };
5925 let head_name = Path::new(head)
5926 .file_name()
5927 .and_then(|name| name.to_str())
5928 .unwrap_or(head);
5929 if !(head_name == "gh" || head_name.eq_ignore_ascii_case("gh.exe")) {
5930 return false;
5931 }
5932 tokens.iter().any(|token| {
5933 matches!(token.as_str(), "--json" | "--jq" | "--template")
5934 || token.starts_with("--json=")
5935 || token.starts_with("--jq=")
5936 || token.starts_with("--template=")
5937 })
5938}
5939
5940fn shell_words_for_flags(command: &str) -> Vec<String> {
5941 let mut words = Vec::new();
5942 let mut current = String::new();
5943 let mut in_single = false;
5944 let mut in_double = false;
5945 let mut escaped = false;
5946
5947 for ch in command.chars() {
5948 if escaped {
5949 current.push(ch);
5950 escaped = false;
5951 continue;
5952 }
5953 if ch == '\\' && !in_single {
5954 escaped = true;
5955 continue;
5956 }
5957 if ch == '\'' && !in_double {
5958 in_single = !in_single;
5959 continue;
5960 }
5961 if ch == '"' && !in_single {
5962 in_double = !in_double;
5963 continue;
5964 }
5965 if ch.is_whitespace() && !in_single && !in_double {
5966 if !current.is_empty() {
5967 words.push(std::mem::take(&mut current));
5968 }
5969 continue;
5970 }
5971 if matches!(ch, ';' | '&' | '|') && !in_single && !in_double {
5972 if !current.is_empty() {
5973 words.push(std::mem::take(&mut current));
5974 }
5975 continue;
5976 }
5977 current.push(ch);
5978 }
5979 if !current.is_empty() {
5980 words.push(current);
5981 }
5982 words
5983}
5984
5985fn is_structured_body(body: &str) -> bool {
5986 let trimmed = body.trim();
5987 if trimmed.is_empty() {
5988 return false;
5989 }
5990 if serde_json::from_str::<serde_json::Value>(trimmed).is_ok() {
5991 return true;
5992 }
5993
5994 let mut saw_line = false;
5995 for line in trimmed
5996 .lines()
5997 .map(str::trim)
5998 .filter(|line| !line.is_empty())
5999 {
6000 saw_line = true;
6001 if serde_json::from_str::<serde_json::Value>(line).is_err() {
6002 return false;
6003 }
6004 }
6005 saw_line
6006}
6007
6008fn stream_starts_like_json(buffer: &BgBuffer, stream: StreamKind) -> bool {
6009 buffer
6010 .read_stream_bounded(stream, 512)
6011 .text
6012 .chars()
6013 .find(|ch| !ch.is_whitespace())
6014 .is_some_and(|ch| matches!(ch, '{' | '[' | '"' | '-' | '0'..='9' | 't' | 'f' | 'n'))
6015}
6016
6017struct CompletionTokenCounts {
6018 original_tokens: Option<u32>,
6019 compressed_tokens: Option<u32>,
6020 original_bytes: Option<i64>,
6021 compressed_bytes: Option<i64>,
6022 tokens_skipped: bool,
6023}
6024
6025impl CompletionTokenCounts {
6026 fn skipped() -> Self {
6027 Self {
6028 original_tokens: None,
6029 compressed_tokens: None,
6030 original_bytes: None,
6031 compressed_bytes: None,
6032 tokens_skipped: true,
6033 }
6034 }
6035}
6036
6037fn live_descendants_summary(metadata: &PersistedTask) -> Option<String> {
6038 let Some(members) = metadata.live_descendants.as_deref() else {
6039 #[cfg(windows)]
6040 return (metadata.mode == BgMode::Pipes).then(|| {
6041 "live descendant check n/a on Windows (background tasks do not use a Job Object)"
6042 .to_string()
6043 });
6044 #[cfg(not(windows))]
6045 return None;
6046 };
6047 let total = members.len() + metadata.live_descendants_omitted;
6048 if total == 0 {
6049 return None;
6050 }
6051
6052 let mut counts = BTreeMap::<&str, usize>::new();
6053 for member in members {
6054 *counts.entry(member.comm.as_str()).or_default() += 1;
6055 }
6056 let mut processes = counts
6057 .into_iter()
6058 .map(|(comm, count)| {
6059 if count == 1 {
6060 comm.to_string()
6061 } else {
6062 format!("{comm} ×{count}")
6063 }
6064 })
6065 .collect::<Vec<_>>()
6066 .join(", ");
6067 if metadata.live_descendants_omitted > 0 {
6068 processes.push_str(&format!(" +{} more", metadata.live_descendants_omitted));
6069 }
6070 let workload = ["vitest", "jest", "cargo test"]
6071 .into_iter()
6072 .find(|name| metadata.command.contains(name))
6073 .map(|name| format!(" ({name})"))
6074 .unwrap_or_default();
6075 Some(format!(
6076 "{total} live descendants still running: {processes}{workload} — they keep the task's process group; bash_kill({}) stops them",
6077 metadata.task_id
6078 ))
6079}
6080
6081fn completion_status_text(status: &BgTaskStatus, exit_code: Option<i32>) -> String {
6082 match status {
6083 BgTaskStatus::TimedOut => "timed out".to_string(),
6084 BgTaskStatus::Killed => "killed".to_string(),
6085 _ => exit_code
6086 .map(|code| format!("exit {code}"))
6087 .unwrap_or_else(|| format!("{status:?}").to_lowercase()),
6088 }
6089}
6090
6091fn token_count_u32(text: &str) -> u32 {
6092 aft_tokenizer::count_tokens(text)
6093 .try_into()
6094 .unwrap_or(u32::MAX)
6095}
6096
6097impl Default for BgTaskRegistry {
6098 fn default() -> Self {
6099 Self::new(Arc::new(Mutex::new(None)))
6100 }
6101}
6102
6103fn modified_within(path: &Path, grace: Duration) -> bool {
6104 fs::metadata(path)
6105 .and_then(|metadata| metadata.modified())
6106 .ok()
6107 .and_then(|modified| SystemTime::now().duration_since(modified).ok())
6108 .map(|age| age < grace)
6109 .unwrap_or(false)
6110}
6111
6112fn canonicalized_path(path: &Path) -> PathBuf {
6113 fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf())
6114}
6115
6116fn started_instant_from_unix_millis(started_at: u64) -> Instant {
6117 let now_ms = SystemTime::now()
6118 .duration_since(UNIX_EPOCH)
6119 .ok()
6120 .map(|duration| duration.as_millis() as u64)
6121 .unwrap_or(started_at);
6122 let elapsed_ms = now_ms.saturating_sub(started_at);
6123 Instant::now()
6124 .checked_sub(Duration::from_millis(elapsed_ms))
6125 .unwrap_or_else(Instant::now)
6126}
6127
6128fn gc_quarantine(storage_dir: &Path) {
6129 let quarantine_root = storage_dir.join("bash-tasks-quarantine");
6130 let Ok(session_dirs) = fs::read_dir(&quarantine_root) else {
6131 return;
6132 };
6133 for session_entry in session_dirs.flatten() {
6134 let session_quarantine_dir = session_entry.path();
6135 if !session_quarantine_dir.is_dir() {
6136 continue;
6137 }
6138 let entries = match fs::read_dir(&session_quarantine_dir) {
6139 Ok(entries) => entries,
6140 Err(error) => {
6141 crate::slog_warn!(
6142 "failed to read background task quarantine dir {}: {error}",
6143 session_quarantine_dir.display()
6144 );
6145 continue;
6146 }
6147 };
6148 for entry in entries.flatten() {
6149 let path = entry.path();
6150 if modified_within(&path, QUARANTINE_GC_GRACE) {
6151 continue;
6152 }
6153 let result = if path.is_dir() {
6154 fs::remove_dir_all(&path)
6155 } else {
6156 fs::remove_file(&path)
6157 };
6158 match result {
6159 Ok(()) => log::debug!(
6160 "deleted old background task quarantine entry {}",
6161 path.display()
6162 ),
6163 Err(error) => crate::slog_warn!(
6164 "failed to delete old background task quarantine entry {}: {error}",
6165 path.display()
6166 ),
6167 }
6168 }
6169 let _ = fs::remove_dir(&session_quarantine_dir);
6170 }
6171 let _ = fs::remove_dir(&quarantine_root);
6172}
6173
6174fn read_for_token_count_from_disk(
6175 metadata: &PersistedTask,
6176 paths: &TaskPaths,
6177 max_bytes_per_stream: usize,
6178) -> TokenCountInput {
6179 if metadata.mode == BgMode::Pty {
6180 return TokenCountInput::Skipped;
6181 }
6182 let stdout = read_file_tail_capped(paths, TaskArtifact::Stdout, max_bytes_per_stream);
6189 let stderr = read_file_tail_capped(paths, TaskArtifact::Stderr, max_bytes_per_stream);
6190 match (stdout, stderr) {
6191 (Ok(stdout), Ok(stderr)) => TokenCountInput::Text(combine_streams(
6192 String::from_utf8_lossy(&stdout).as_ref(),
6193 String::from_utf8_lossy(&stderr).as_ref(),
6194 )),
6195 (Ok(stdout), Err(_)) => TokenCountInput::Text(combine_streams(
6196 String::from_utf8_lossy(&stdout).as_ref(),
6197 "",
6198 )),
6199 (Err(_), Ok(stderr)) => TokenCountInput::Text(combine_streams(
6200 "",
6201 String::from_utf8_lossy(&stderr).as_ref(),
6202 )),
6203 (Err(_), Err(_)) => TokenCountInput::Skipped,
6204 }
6205}
6206
6207fn read_file_tail_capped(
6208 paths: &TaskPaths,
6209 artifact: TaskArtifact,
6210 max_bytes: usize,
6211) -> std::io::Result<Vec<u8>> {
6212 let mut file = open_task_artifact(paths, artifact)?;
6213 file.tail(max_bytes).map(|(bytes, _)| bytes)
6214}
6215
6216fn task_bundle_is_absent(storage_dir: &Path, session_id: &str, task_id: &str) -> bool {
6217 let session_dir = session_tasks_dir(storage_dir, session_id);
6218 !session_dir.join(task_id).exists() && !session_dir.join(format!("{task_id}.json")).exists()
6219}
6220
6221fn terminal_db_row_snapshot(row: BashTaskRow, metadata: PersistedTask) -> BgTaskSnapshot {
6222 let existing_path = |path: Option<String>| {
6223 path.filter(|path| {
6224 fs::metadata(path)
6225 .map(|metadata| metadata.is_file())
6226 .unwrap_or(false)
6227 })
6228 };
6229 let duration_ms = metadata.duration_ms.or_else(|| {
6230 metadata
6231 .finished_at
6232 .map(|finished_at| finished_at.saturating_sub(metadata.started_at))
6233 });
6234 let live_descendants_summary = live_descendants_summary(&metadata);
6235 BgTaskSnapshot {
6236 info: BgTaskInfo {
6237 task_id: metadata.task_id,
6238 status: metadata.status,
6239 command: metadata.command,
6240 mode: metadata.mode.clone(),
6241 started_at: metadata.started_at,
6242 duration_ms,
6243 status_reason: metadata.status_reason,
6244 },
6245 exit_code: metadata.exit_code,
6246 child_pid: metadata.child_pid,
6247 workdir: metadata.workdir.display().to_string(),
6248 output_preview: String::new(),
6249 bash_output_list_envelope: None,
6250 output_truncated: false,
6251 output_path: existing_path(row.stdout_path),
6252 stderr_path: existing_path(row.stderr_path),
6253 pty_rows: (metadata.mode == BgMode::Pty).then_some(metadata.pty_rows.unwrap_or(24)),
6254 pty_cols: (metadata.mode == BgMode::Pty).then_some(metadata.pty_cols.unwrap_or(80)),
6255 pty_screen: None,
6256 scanner_report: metadata.scanner_report,
6257 sandbox_native: metadata.sandbox_native,
6258 sandbox_unavailable: false,
6259 live_descendants: metadata.live_descendants.clone(),
6260 live_descendants_omitted: metadata.live_descendants_omitted,
6261 live_descendants_summary,
6262 kill_signaled: false,
6263 kill_reached: 0,
6264 }
6265}
6266
6267impl BgTask {
6268 fn snapshot(&self, preview_bytes: usize) -> BgTaskSnapshot {
6269 let state = self
6270 .state
6271 .lock()
6272 .unwrap_or_else(|poison| poison.into_inner());
6273 self.snapshot_locked(&state, preview_bytes)
6274 }
6275
6276 fn snapshot_locked(&self, state: &BgTaskState, preview_bytes: usize) -> BgTaskSnapshot {
6277 let metadata = &state.metadata;
6278 let duration_ms = metadata.duration_ms.or_else(|| {
6279 metadata
6280 .status
6281 .is_terminal()
6282 .then(|| self.started.elapsed().as_millis() as u64)
6283 });
6284 let (output_preview, output_truncated) = if metadata.mode == BgMode::Pty {
6285 (String::new(), false)
6286 } else if metadata.status.is_terminal() {
6287 state
6288 .terminal_output_cache
6289 .as_ref()
6290 .map(|cache| (cache.output_preview.clone(), cache.output_truncated))
6291 .unwrap_or_else(|| (String::new(), false))
6292 } else if preview_bytes == 0 {
6293 (String::new(), false)
6294 } else {
6295 state.buffer.read_tail(preview_bytes)
6296 };
6297 BgTaskSnapshot {
6298 info: BgTaskInfo {
6299 task_id: self.task_id.clone(),
6300 status: metadata.status.clone(),
6301 command: metadata.command.clone(),
6302 mode: metadata.mode.clone(),
6303 started_at: metadata.started_at,
6304 duration_ms,
6305 status_reason: metadata.status_reason.clone(),
6306 },
6307 exit_code: metadata.exit_code,
6308 child_pid: metadata.child_pid,
6309 workdir: metadata.workdir.display().to_string(),
6310 output_preview,
6311 bash_output_list_envelope: None,
6312 output_truncated,
6313 output_path: state
6314 .buffer
6315 .output_path()
6316 .map(|path| path.display().to_string()),
6317 stderr_path: state
6318 .buffer
6319 .stderr_path()
6320 .map(|path| path.display().to_string()),
6321 pty_rows: (metadata.mode == BgMode::Pty).then_some(metadata.pty_rows.unwrap_or(24)),
6322 pty_cols: (metadata.mode == BgMode::Pty).then_some(metadata.pty_cols.unwrap_or(80)),
6323 pty_screen: None,
6324 scanner_report: metadata.scanner_report.clone(),
6325 sandbox_native: metadata.sandbox_native,
6326 sandbox_unavailable: metadata.sandbox_native
6327 && open_task_artifact(&self.paths, TaskArtifact::SandboxUnavailable)
6328 .and_then(|mut file| file.read_all())
6329 .is_ok_and(|bytes| bytes == b"sandbox_unavailable"),
6330 live_descendants: metadata.live_descendants.clone(),
6331 live_descendants_omitted: metadata.live_descendants_omitted,
6332 live_descendants_summary: live_descendants_summary(metadata),
6333 kill_signaled: false,
6334 kill_reached: 0,
6335 }
6336 }
6337
6338 pub(crate) fn is_running(&self) -> bool {
6339 self.state
6340 .lock()
6341 .map(|state| {
6342 state.metadata.status == BgTaskStatus::Running
6343 || (state.metadata.mode == BgMode::Pty
6344 && state.metadata.status == BgTaskStatus::Killing)
6345 })
6346 .unwrap_or(false)
6347 }
6348
6349 fn is_terminal(&self) -> bool {
6350 self.state
6351 .lock()
6352 .map(|state| state.metadata.status.is_terminal())
6353 .unwrap_or(false)
6354 }
6355
6356 fn mark_terminal_now(&self) {
6357 if let Ok(mut terminal_at) = self.terminal_at.lock() {
6358 if terminal_at.is_none() {
6359 *terminal_at = Some(Instant::now());
6360 }
6361 }
6362 }
6363
6364 fn set_completion_delivered(
6365 &self,
6366 delivered: bool,
6367 registry: &BgTaskRegistry,
6368 ) -> std::io::Result<()> {
6369 let mut state = self
6370 .state
6371 .lock()
6372 .map_err(|_| std::io::Error::other("background task lock poisoned"))?;
6373 let updated = registry.update_task_metadata(&self.paths, |metadata| {
6374 metadata.completion_delivered = delivered;
6375 })?;
6376 state.metadata = updated;
6377 Ok(())
6378 }
6379}
6380
6381#[cfg(unix)]
6402fn reap_piped_child(child_slot: &mut Option<Child>) {
6403 if let Some(mut child) = child_slot.take() {
6404 if matches!(child.try_wait(), Ok(None)) {
6405 let _ = child.wait();
6406 }
6407 }
6408}
6409
6410#[cfg(windows)]
6415fn reap_piped_child(child_slot: &mut Option<Child>) {
6416 *child_slot = None;
6417}
6418
6419fn terminal_metadata_from_marker(
6420 mut metadata: PersistedTask,
6421 marker: ExitMarker,
6422 reason: Option<String>,
6423) -> PersistedTask {
6424 match marker {
6425 ExitMarker::Code(code) => {
6426 let status = if code == 0 {
6427 BgTaskStatus::Completed
6428 } else {
6429 BgTaskStatus::Failed
6430 };
6431 metadata.mark_terminal(status, Some(code), reason);
6432 }
6433 ExitMarker::Killed => metadata.mark_terminal(
6434 BgTaskStatus::Killed,
6435 terminal_exit_code_for_status(&BgTaskStatus::Killed),
6436 reason,
6437 ),
6438 }
6439 metadata
6440}
6441
6442fn terminal_exit_code_for_status(status: &BgTaskStatus) -> Option<i32> {
6443 match status {
6444 BgTaskStatus::TimedOut => Some(124),
6445 BgTaskStatus::Killed => Some(137),
6446 _ => None,
6447 }
6448}
6449
6450fn attach_sandbox_metadata(metadata: &mut PersistedTask, spawn_plan: &SpawnPlan) {
6451 metadata.sandbox_native = spawn_plan.is_native_launcher();
6452 metadata.sandbox_temp_dir = spawn_plan.temp_dir().map(Path::to_path_buf);
6453}
6454
6455#[cfg(unix)]
6456pub(crate) fn resolve_posix_shell() -> PathBuf {
6457 static POSIX_SHELL: OnceLock<PathBuf> = OnceLock::new();
6458 POSIX_SHELL
6459 .get_or_init(|| {
6460 std::env::var_os("BASH")
6461 .filter(|value| !value.is_empty())
6462 .map(PathBuf::from)
6463 .filter(|path| path.exists())
6464 .or_else(|| which::which("bash").ok())
6465 .or_else(|| which::which("zsh").ok())
6466 .unwrap_or_else(|| PathBuf::from("/bin/sh"))
6467 })
6468 .clone()
6469}
6470
6471#[cfg(windows)]
6472fn detached_shell_command_for(
6473 shell: crate::windows_shell::WindowsShell,
6474 command: &str,
6475 exit_path: &Path,
6476 paths: &TaskPaths,
6477 creation_flags: u32,
6478) -> Result<Command, String> {
6479 use crate::windows_shell::WindowsShell;
6480 let wrapper_body = shell.wrapper_script_bytes(command, exit_path);
6493 let wrapper_ext = match shell {
6494 WindowsShell::Pwsh | WindowsShell::Powershell => "ps1",
6495 WindowsShell::Cmd => "bat",
6496 WindowsShell::Posix(_) => "sh",
6500 };
6501 let wrapper_path = paths.dir.join(format!(
6502 "{}.{}",
6503 paths
6504 .json
6505 .file_stem()
6506 .and_then(|s| s.to_str())
6507 .unwrap_or("wrapper"),
6508 wrapper_ext
6509 ));
6510 fs::write(&wrapper_path, wrapper_body)
6511 .map_err(|e| format!("failed to write background bash wrapper script: {e}"))?;
6512
6513 let mut cmd = Command::new(shell.binary().as_ref());
6514 match shell {
6515 WindowsShell::Pwsh | WindowsShell::Powershell => {
6516 cmd.args([
6519 "-NoLogo",
6520 "-NoProfile",
6521 "-NonInteractive",
6522 "-ExecutionPolicy",
6523 "Bypass",
6524 "-File",
6525 ]);
6526 cmd.arg(&wrapper_path);
6527 }
6528 WindowsShell::Cmd => {
6529 cmd.args(["/D", "/C"]);
6536 cmd.arg(&wrapper_path);
6537 }
6538 WindowsShell::Posix(_) => {
6539 cmd.arg(&wrapper_path);
6544 }
6545 }
6546
6547 cmd.creation_flags(creation_flags);
6551 Ok(cmd)
6552}
6553
6554fn spawn_detached_child(
6570 spawn_plan: &SpawnPlan,
6571 command: &str,
6572 shell: super::BashShell,
6573 shell_path: &Path,
6574 paths: &TaskPaths,
6575 workdir: &Path,
6576 env: &HashMap<String, String>,
6577 io_handles: &mut TaskIoHandles,
6578 capture_pipeline_status: bool,
6579 #[cfg_attr(not(target_os = "linux"), allow(unused_variables))] linux_scope: bool,
6580) -> Result<std::process::Child, String> {
6581 #[cfg(windows)]
6582 let _ = capture_pipeline_status;
6583 #[cfg(not(windows))]
6584 let _ = (command, shell);
6585 #[cfg(not(windows))]
6586 {
6587 use std::os::fd::AsRawFd;
6588
6589 let stdout = io_handles
6590 .clone_file(TaskArtifact::Stdout)
6591 .map_err(|e| format!("failed to clone stdout capture handle: {e}"))?;
6592 let stderr = io_handles
6593 .clone_file(TaskArtifact::Stderr)
6594 .map_err(|e| format!("failed to clone stderr capture handle: {e}"))?;
6595 let prepared = spawn_plan
6596 .prepared_task()
6597 .ok_or_else(|| "background task payload was not prepared".to_string())?;
6598 let payload = prepared.invocation()?;
6599 let exit = io_handles
6600 .inheritable_file(TaskArtifact::Exit)
6601 .map_err(|e| format!("failed to inherit exit marker handle: {e}"))?;
6602 let failure = io_handles
6603 .inheritable_file(TaskArtifact::SandboxUnavailable)
6604 .map_err(|e| format!("failed to inherit sandbox failure marker handle: {e}"))?;
6605 let pipeline_status = capture_pipeline_status
6606 .then(|| io_handles.inheritable_file(TaskArtifact::PipelineStatus))
6607 .transpose()
6608 .map_err(|e| format!("failed to inherit pipeline status handle: {e}"))?;
6609 let shell_path = spawn_plan.host_shell_path().unwrap_or(shell_path);
6610 let pipeline_shell = super::process::pipeline_shell_kind(shell_path).unwrap_or("");
6611 let pipeline_status_fd = if capture_pipeline_status {
6612 crate::sandbox_spawn::CHILD_PIPE_STATUS_FD.to_string()
6613 } else {
6614 String::new()
6615 };
6616 let base_executable = PathBuf::from("/bin/sh");
6617 let base_args = vec![
6618 OsString::from("-c"),
6619 payload.wrapper_text.clone(),
6620 OsString::from("aft-payload-wrapper"),
6621 shell_path.as_os_str().to_os_string(),
6622 payload.command_text.clone(),
6623 OsString::from(crate::sandbox_spawn::CHILD_EXIT_FD.to_string()),
6624 OsString::from(pipeline_status_fd),
6625 OsString::from(pipeline_shell),
6626 ];
6627 #[cfg(target_os = "linux")]
6628 let (executable, args) = if linux_scope {
6629 if spawn_plan.is_native_launcher() {
6630 crate::slog_info!(
6631 "bash.linux_scope requested but native sandbox launch cannot reach the user manager; spawning without a systemd scope"
6632 );
6633 (base_executable, base_args)
6634 } else if let Some(launcher) = super::process::select_systemd_scope_launcher() {
6635 super::process::systemd_scope_argv(&launcher, &base_executable, &base_args)
6636 } else {
6637 crate::slog_info!(
6638 "bash.linux_scope requested but systemd-run or the user manager is unavailable; spawning without a systemd scope"
6639 );
6640 (base_executable, base_args)
6641 }
6642 } else {
6643 (base_executable, base_args)
6644 };
6645 #[cfg(not(target_os = "linux"))]
6646 let (executable, args) = {
6647 let _ = linux_scope;
6648 (base_executable, base_args)
6649 };
6650 let (mut child_command, profile_handle) = crate::sandbox_spawn::detached_command_for_plan(
6651 spawn_plan,
6652 executable.as_os_str(),
6653 &args,
6654 &paths.json,
6655 crate::sandbox_spawn::CHILD_EXIT_FD,
6656 crate::sandbox_spawn::CHILD_FAILURE_FD,
6657 )?;
6658 crate::sandbox_spawn::apply_marker_fd_allowlist(
6659 &mut child_command,
6660 exit.as_raw_fd(),
6661 failure.as_raw_fd(),
6662 pipeline_status.as_ref().map(|file| file.as_raw_fd()),
6663 )?;
6664 child_command
6665 .current_dir(workdir)
6666 .stdin(Stdio::null())
6667 .stdout(Stdio::from(stdout))
6668 .stderr(Stdio::from(stderr));
6669 crate::agent_child_env::apply_to_command(&mut child_command, env);
6670 crate::sandbox_spawn::apply_sandbox_environment(spawn_plan, &mut child_command, env);
6671 let child = child_command
6672 .spawn()
6673 .map_err(|e| format!("failed to spawn background bash command: {e}"));
6674 drop((payload, exit, failure, pipeline_status, profile_handle));
6675 child
6676 }
6677 #[cfg(windows)]
6678 {
6679 let _ = shell_path;
6680 use crate::windows_shell::shell_candidates;
6681 match spawn_plan {
6682 SpawnPlan::Unsandboxed | SpawnPlan::Host { .. } => {}
6683 SpawnPlan::Refused { code, .. } => return Err((*code).to_string()),
6684 SpawnPlan::Launcher { .. } => return Err("sandbox_unavailable".to_string()),
6685 }
6686 let candidates: Vec<crate::windows_shell::WindowsShell> = if shell.is_powershell() {
6697 vec![crate::windows_shell::WindowsShell::Pwsh]
6698 } else {
6699 shell_candidates()
6700 };
6701 const FLAG_CREATE_NEW_PROCESS_GROUP: u32 = 0x0000_0200;
6714 const FLAG_CREATE_BREAKAWAY_FROM_JOB: u32 = 0x0100_0000;
6715 const FLAG_CREATE_NO_WINDOW: u32 = 0x0800_0000;
6716 let with_breakaway =
6717 FLAG_CREATE_NO_WINDOW | FLAG_CREATE_NEW_PROCESS_GROUP | FLAG_CREATE_BREAKAWAY_FROM_JOB;
6718 let without_breakaway = FLAG_CREATE_NO_WINDOW | FLAG_CREATE_NEW_PROCESS_GROUP;
6719 let mut last_error: Option<String> = None;
6720 for (idx, shell) in candidates.iter().enumerate() {
6721 for &flags in &[with_breakaway, without_breakaway] {
6725 let stdout = io_handles
6728 .clone_file(TaskArtifact::Stdout)
6729 .map_err(|e| format!("failed to clone stdout capture handle: {e}"))?;
6730 let stderr = io_handles
6731 .clone_file(TaskArtifact::Stderr)
6732 .map_err(|e| format!("failed to clone stderr capture handle: {e}"))?;
6733 let mut cmd =
6734 detached_shell_command_for(shell.clone(), command, &paths.exit, paths, flags)?;
6735 cmd.current_dir(workdir)
6736 .stdin(Stdio::null())
6737 .stdout(Stdio::from(stdout))
6738 .stderr(Stdio::from(stderr));
6739 crate::agent_child_env::apply_to_command(&mut cmd, env);
6740 match cmd.spawn() {
6741 Ok(child) => {
6742 if idx > 0 {
6743 crate::slog_warn!("background bash spawn fell back to {} after {} earlier candidate(s) failed; \
6744 the cached PATH probe disagreed with runtime spawn — likely PATH \
6745 inheritance, antivirus / AppLocker / Defender ASR, or sandbox policy.",
6746 shell.binary(),
6747 idx);
6748 }
6749 if flags == without_breakaway {
6750 crate::slog_warn!(
6751 "background bash spawn: CREATE_BREAKAWAY_FROM_JOB rejected \
6752 (likely a restrictive Job Object — CI sandbox or MDM policy). \
6753 Spawned without breakaway; the bg task will be torn down if the \
6754 AFT process group is killed."
6755 );
6756 }
6757 return Ok(child);
6758 }
6759 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
6760 crate::slog_warn!("background bash spawn: {} returned NotFound at runtime — trying next candidate",
6761 shell.binary());
6762 last_error = Some(format!("{}: {e}", shell.binary()));
6763 break;
6766 }
6767 Err(e) if flags == with_breakaway && e.raw_os_error() == Some(5) => {
6768 crate::slog_warn!(
6770 "background bash spawn: CREATE_BREAKAWAY_FROM_JOB rejected with \
6771 Access Denied — retrying {} without breakaway",
6772 shell.binary()
6773 );
6774 last_error = Some(format!("{}: {e}", shell.binary()));
6775 continue;
6776 }
6777 Err(e) => {
6778 return Err(format!(
6779 "failed to spawn background bash command via {}: {e}",
6780 shell.binary()
6781 ));
6782 }
6783 }
6784 }
6785 }
6786 Err(format!(
6787 "failed to spawn background bash command: no Windows shell could be spawned. \
6788 Last error: {}. PATH-probed candidates: {:?}",
6789 last_error.unwrap_or_else(|| "no candidates were attempted".to_string()),
6790 candidates.iter().map(|s| s.binary()).collect::<Vec<_>>()
6791 ))
6792 }
6793}
6794
6795#[cfg(test)]
6796fn random_slug() -> String {
6797 let mut bytes = [0u8; 8];
6805 getrandom::fill(&mut bytes).unwrap_or_else(|_| {
6807 let t = SystemTime::now()
6809 .duration_since(UNIX_EPOCH)
6810 .map(|d| d.as_nanos() as u64)
6811 .unwrap_or(0);
6812 let p = u64::from(std::process::id());
6813 bytes.copy_from_slice(&(t ^ p.rotate_left(32)).to_le_bytes());
6814 });
6815 let hex: String = bytes.iter().map(|b| format!("{b:02x}")).collect();
6817 format!("bash-{hex}")
6818}
6819
6820#[cfg(test)]
6821mod tests {
6822 use std::collections::HashMap;
6823 use std::fs;
6824 use std::io::Write;
6825 #[cfg(unix)]
6826 use std::os::unix::fs::PermissionsExt;
6827 use std::sync::atomic::{AtomicBool, AtomicUsize};
6828 use std::sync::{Arc, Mutex};
6829 use std::time::{Duration, Instant, SystemTime};
6830
6831 use super::*;
6832 use crate::bash_background::persistence::{read_task, task_paths, write_task};
6833
6834 #[cfg(unix)]
6835 const QUICK_SUCCESS_COMMAND: &str = "true";
6836 #[cfg(windows)]
6837 const QUICK_SUCCESS_COMMAND: &str = "cmd /c exit 0";
6838
6839 const CHILD_EXIT_LIVENESS_BOUND: Duration = Duration::from_secs(30);
6844
6845 #[cfg(unix)]
6846 const LONG_RUNNING_COMMAND: &str = "sleep 5";
6847
6848 #[cfg(unix)]
6849 #[test]
6850 fn launcher_plans_disable_pipeline_status_capture() {
6851 let launcher = SpawnPlan::launcher_for_test(
6852 crate::sandbox_profile::SandboxProfile {
6853 v: crate::sandbox_profile::SANDBOX_PROFILE_VERSION,
6854 writable_roots: Vec::new(),
6855 write_deny: Vec::new(),
6856 write_deny_nested: Vec::new(),
6857 read_allow: Vec::new(),
6858 read_deny: Vec::new(),
6859 socket_deny: Vec::new(),
6860 cache_roots: Vec::new(),
6861 temp_dir: PathBuf::from("/tmp/aft-test-sandbox"),
6862 },
6863 PathBuf::from("/bin/true"),
6864 );
6865 assert!(!should_capture_pipeline_status(
6866 &launcher,
6867 true,
6868 Path::new("/bin/bash")
6869 ));
6870 assert!(should_capture_pipeline_status(
6871 &SpawnPlan::Unsandboxed,
6872 true,
6873 Path::new("/bin/bash")
6874 ));
6875 }
6876
6877 #[cfg(windows)]
6878 const LONG_RUNNING_COMMAND: &str = "cmd /c timeout /t 5 /nobreak > nul";
6879
6880 #[test]
6881 fn bash_memory_estimate_is_zero_when_empty_and_nonzero_for_completion_cache() {
6882 let registry = BgTaskRegistry::default();
6883 assert_eq!(registry.estimated_memory().estimated_bytes, Some(0));
6884 registry
6885 .inner
6886 .completions
6887 .lock()
6888 .unwrap()
6889 .push_back(BgCompletion {
6890 task_id: "bash-memory".to_string(),
6891 session_id: "session-memory".to_string(),
6892 status: BgTaskStatus::Completed,
6893 exit_code: Some(0),
6894 command: "printf memory".to_string(),
6895 output_preview: "resident completion output".to_string(),
6896 bash_output_list_envelope: None,
6897 output_truncated: false,
6898 original_tokens: None,
6899 compressed_tokens: None,
6900 tokens_skipped: false,
6901 status_reason: None,
6902 live_descendants: Some(Vec::new()),
6903 live_descendants_omitted: 0,
6904 live_descendants_summary: None,
6905 });
6906 let estimate = registry.estimated_memory();
6907 assert!(estimate.estimated_bytes.unwrap() > 0);
6908 assert_eq!(estimate.counts["completion_caches"], 1);
6909 assert_eq!(estimate.counts["sessions"], 1);
6910 }
6911
6912 #[test]
6913 fn gh_structured_detection_rejects_piped_commands() {
6914 assert!(is_gh_structured_command(
6915 "gh issue list --json number,title"
6916 ));
6917 assert!(is_gh_structured_command(
6918 "cd repo && gh issue list --json number,title"
6919 ));
6920
6921 assert!(!is_gh_structured_command(
6922 "gh issue list --json number,title | jq '.[]'"
6923 ));
6924 assert!(!is_gh_structured_command(
6925 "gh issue list --json number,title |"
6926 ));
6927 }
6928
6929 fn insert_terminal_piped_task(
6930 registry: &BgTaskRegistry,
6931 dir: &tempfile::TempDir,
6932 command: &str,
6933 stdout: &str,
6934 stderr: &str,
6935 compressed: bool,
6936 ) -> (String, Arc<BgTask>) {
6937 let task_id = random_slug();
6938 let paths = task_paths(dir.path(), "session", &task_id).unwrap();
6939 fs::create_dir_all(&paths.dir).unwrap();
6940 fs::write(&paths.stdout, stdout).unwrap();
6941 fs::write(&paths.stderr, stderr).unwrap();
6942 let mut metadata = PersistedTask::starting(
6943 task_id.clone(),
6944 "session".to_string(),
6945 command.to_string(),
6946 dir.path().to_path_buf(),
6947 Some(dir.path().to_path_buf()),
6948 Some(30_000),
6949 true,
6950 compressed,
6951 );
6952 metadata.mark_terminal(BgTaskStatus::Completed, Some(0), None);
6953 write_task(&paths.json, &metadata).unwrap();
6954 registry
6955 .insert_rehydrated_task(metadata, paths, true)
6956 .expect("insert terminal task");
6957 let task = registry.task_for_session(&task_id, "session").unwrap();
6958 (task_id, task)
6959 }
6960
6961 #[test]
6962 fn bash_zero_preview_running_status_skips_output_read_while_explicit_preview_reads() {
6963 let registry = BgTaskRegistry::default();
6964 let dir = tempfile::tempdir().unwrap();
6965 let task_id = random_slug();
6966 let paths = task_paths(dir.path(), "session", &task_id).unwrap();
6967 fs::create_dir_all(&paths.dir).unwrap();
6968 fs::write(&paths.stdout, "live output\n").unwrap();
6969 fs::write(&paths.stderr, "").unwrap();
6970 let stdout_path = paths.stdout.clone();
6971 let mut metadata = PersistedTask::starting(
6972 task_id.clone(),
6973 "session".to_string(),
6974 "sleep 60".to_string(),
6975 dir.path().to_path_buf(),
6976 Some(dir.path().to_path_buf()),
6977 Some(30_000),
6978 true,
6979 false,
6980 );
6981 metadata.status = BgTaskStatus::Running;
6982 write_task(&paths.json, &metadata).unwrap();
6983 registry
6984 .insert_rehydrated_task(metadata, paths, false)
6985 .expect("insert running task");
6986
6987 crate::bash_background::buffer::reset_tail_read_count(&stdout_path);
6988 for _ in 0..5 {
6989 let snapshot = registry
6990 .status(&task_id, "session", Some(dir.path()), Some(dir.path()), 0)
6991 .expect("running snapshot");
6992 assert_eq!(snapshot.info.status, BgTaskStatus::Running);
6993 assert!(snapshot.output_preview.is_empty());
6994 }
6995 assert_eq!(
6996 crate::bash_background::buffer::tail_read_count(&stdout_path),
6997 0
6998 );
6999
7000 let snapshot = registry
7001 .status(
7002 &task_id,
7003 "session",
7004 Some(dir.path()),
7005 Some(dir.path()),
7006 RUNNING_OUTPUT_PREVIEW_BYTES,
7007 )
7008 .expect("explicit running snapshot");
7009 assert_eq!(snapshot.output_preview, "live output\n");
7010 assert_eq!(
7011 crate::bash_background::buffer::tail_read_count(&stdout_path),
7012 1
7013 );
7014 }
7015
7016 #[test]
7017 fn artifact_read_capability_requires_exact_canonical_path_and_session() {
7018 let registry = BgTaskRegistry::default();
7019 let dir = tempfile::tempdir().unwrap();
7020 let (_task_id, task) = insert_terminal_piped_task(
7021 ®istry,
7022 &dir,
7023 "printf output",
7024 "stdout\n",
7025 "stderr\n",
7026 true,
7027 );
7028 fs::write(&task.paths.exit, "0\n").unwrap();
7029
7030 assert!(registry.is_session_owned_artifact_path("session", &task.paths.stdout));
7031 assert!(registry.is_session_owned_artifact_path("session", &task.paths.stderr));
7032 assert!(registry.is_session_owned_artifact_path("session", &task.paths.exit));
7033 assert!(!registry.is_session_owned_artifact_path("different-session", &task.paths.stdout));
7034 assert!(!registry.is_session_owned_artifact_path("session", &task.paths.json));
7035
7036 let unregistered = task.paths.dir.join("unregistered-output");
7037 fs::write(&unregistered, "not a task artifact\n").unwrap();
7038 assert!(!registry.is_session_owned_artifact_path("session", &unregistered));
7039 }
7040
7041 #[cfg(unix)]
7042 #[test]
7043 fn artifact_directory_symlink_does_not_create_a_prefix_exception() {
7044 let registry = BgTaskRegistry::default();
7045 let dir = tempfile::tempdir().unwrap();
7046 let project = dir.path().join("project");
7047 fs::create_dir_all(&project).unwrap();
7048 let (_task_id, task) =
7049 insert_terminal_piped_task(®istry, &dir, "printf output", "stdout\n", "", true);
7050 let link = project.join("task-artifacts");
7051 std::os::unix::fs::symlink(&task.paths.dir, &link).unwrap();
7052 let unregistered = task.paths.dir.join("unregistered-output");
7053 fs::write(&unregistered, "not registered\n").unwrap();
7054
7055 assert!(!registry.is_session_owned_artifact_path("session", &link));
7056 assert!(
7057 !registry.is_session_owned_artifact_path("session", &link.join("unregistered-output"))
7058 );
7059 assert!(registry.is_session_owned_artifact_path(
7060 "session",
7061 &link.join(task.paths.stdout.file_name().unwrap())
7062 ));
7063
7064 let outside = dir.path().join("outside-secret");
7065 fs::write(&outside, "must stay private\n").unwrap();
7066 fs::remove_file(&task.paths.stdout).unwrap();
7067 std::os::unix::fs::symlink(&outside, &task.paths.stdout).unwrap();
7068 assert!(!registry.is_session_owned_artifact_path("session", &task.paths.stdout));
7069 }
7070
7071 #[test]
7072 fn recovery_footer_uses_bash_status_when_artifact_is_not_registered() {
7073 let registry = BgTaskRegistry::default();
7074 let dir = tempfile::tempdir().unwrap();
7075 let task_id = "bash-1111111111111111";
7076 let paths = task_paths(dir.path(), "session", task_id).unwrap();
7077 fs::create_dir_all(&paths.dir).unwrap();
7078 fs::write(
7079 &paths.stdout,
7080 format!("{}tail\n", "output-line\n".repeat(2_000)),
7081 )
7082 .unwrap();
7083 fs::write(&paths.stderr, "").unwrap();
7084 let mut metadata = PersistedTask::starting(
7085 task_id.to_string(),
7086 "session".to_string(),
7087 "printf output".to_string(),
7088 dir.path().to_path_buf(),
7089 Some(dir.path().to_path_buf()),
7090 Some(30_000),
7091 true,
7092 true,
7093 );
7094 metadata.mark_terminal(BgTaskStatus::Completed, Some(0), None);
7095 write_task(&paths.json, &metadata).unwrap();
7096
7097 let cache = registry
7098 .render_terminal_output_from_paths(&metadata, &paths)
7099 .expect("terminal render");
7100
7101 assert!(cache
7102 .output_preview
7103 .contains("use bash_status({taskId: \"bash-1111111111111111\"})"));
7104 assert!(!cache.output_preview.contains("full output: read "));
7105 }
7106
7107 fn insert_terminal_pty_task(
7108 registry: &BgTaskRegistry,
7109 dir: &tempfile::TempDir,
7110 pty_output: &str,
7111 ) -> (String, Arc<BgTask>) {
7112 let task_id = random_slug();
7113 let paths = task_paths(dir.path(), "session", &task_id).unwrap();
7114 fs::create_dir_all(&paths.dir).unwrap();
7115 fs::write(&paths.pty, pty_output).unwrap();
7116 let mut metadata = PersistedTask::starting(
7117 task_id.clone(),
7118 "session".to_string(),
7119 "python".to_string(),
7120 dir.path().to_path_buf(),
7121 Some(dir.path().to_path_buf()),
7122 Some(30_000),
7123 true,
7124 true,
7125 );
7126 metadata.mode = BgMode::Pty;
7127 metadata.mark_terminal(BgTaskStatus::Completed, Some(0), None);
7128 write_task(&paths.json, &metadata).unwrap();
7129 registry
7130 .insert_rehydrated_task(metadata, paths, true)
7131 .expect("insert terminal pty task");
7132 let task = registry.task_for_session(&task_id, "session").unwrap();
7133 (task_id, task)
7134 }
7135
7136 #[cfg(unix)]
7137 fn wait_for_terminal_snapshot(
7138 registry: &BgTaskRegistry,
7139 task_id: &str,
7140 session_id: &str,
7141 project: &Path,
7142 storage: &Path,
7143 ) -> BgTaskSnapshot {
7144 let started = Instant::now();
7145 loop {
7146 let snapshot = registry
7147 .status(task_id, session_id, Some(project), Some(storage), 4096)
7148 .expect("spawned task should be visible to status");
7149 if snapshot.info.status.is_terminal() {
7150 return snapshot;
7151 }
7152 assert!(
7153 started.elapsed() < Duration::from_secs(10),
7154 "timed out waiting for task {task_id} to finish; last status={:?}",
7155 snapshot.info.status
7156 );
7157 std::thread::sleep(Duration::from_millis(50));
7158 }
7159 }
7160
7161 fn write_running_project_task(storage: &Path, project: &Path, session: &str, task_id: &str) {
7162 let paths = task_paths(storage, session, task_id).unwrap();
7163 let mut metadata = PersistedTask::starting(
7164 task_id.to_string(),
7165 session.to_string(),
7166 "sleep 60".to_string(),
7167 project.to_path_buf(),
7168 Some(project.to_path_buf()),
7169 Some(30_000),
7170 true,
7171 true,
7172 );
7173 metadata.status = BgTaskStatus::Running;
7174 metadata.child_pid = Some(std::process::id());
7180 write_task(&paths.json, &metadata).unwrap();
7181 fs::write(&paths.stdout, "still running\n").unwrap();
7182 fs::write(&paths.stderr, "").unwrap();
7183 }
7184
7185 #[test]
7186 fn status_replay_filters_same_session_by_project_root() {
7187 let project_a = tempfile::tempdir().unwrap();
7188 let project_b = tempfile::tempdir().unwrap();
7189 let storage = tempfile::tempdir().unwrap();
7190 let session = "shared-session";
7191 let task_id = "bash-2222222222222222";
7192 write_running_project_task(storage.path(), project_a.path(), session, task_id);
7193
7194 let actor_b = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
7195 assert!(actor_b
7196 .status(
7197 task_id,
7198 session,
7199 Some(project_b.path()),
7200 Some(storage.path()),
7201 1024,
7202 )
7203 .is_none());
7204 assert!(actor_b.task_for_session(task_id, session).is_none());
7205
7206 let actor_a = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
7207 let snapshot = actor_a
7208 .status(
7209 task_id,
7210 session,
7211 Some(project_a.path()),
7212 Some(storage.path()),
7213 1024,
7214 )
7215 .expect("owning project should replay its task");
7216 assert_eq!(snapshot.info.status, BgTaskStatus::Running);
7217 }
7218
7219 #[cfg(unix)]
7220 #[test]
7221 fn multiline_pipeline_stdout_persists_all_lines_after_terminal_status() {
7222 let cases = [
7223 (
7224 "long-first",
7225 "sleep 0.5; printf 'one\\n' | cat\nprintf 'two\\n' | grep -c two\nprintf 'three\\n' | cat",
7226 vec!["one", "1", "three"],
7227 ),
7228 (
7229 "short-first",
7230 "printf 'one\\n' | cat\nsleep 0.2; printf 'two\\n' | grep -c two\nprintf 'three\\n' | cat",
7231 vec!["one", "1", "three"],
7232 ),
7233 (
7234 "failing-middle",
7235 "sleep 0.2; printf 'one\\n' | cat\nfalse; printf 'after-false\\n' | cat\nprintf 'three\\n' | cat",
7236 vec!["one", "after-false", "three"],
7237 ),
7238 ];
7239
7240 for (name, command, expected_lines) in cases {
7241 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
7242 let dir = tempfile::tempdir().unwrap();
7243 let session_id = format!("session-{name}");
7244 let task_id = registry
7245 .spawn(
7246 SpawnPlan::Unsandboxed,
7247 command,
7248 session_id.clone(),
7249 dir.path().to_path_buf(),
7250 HashMap::new(),
7251 Some(Duration::from_secs(30)),
7252 dir.path().to_path_buf(),
7253 10,
7254 true,
7255 true,
7256 Some(dir.path().to_path_buf()),
7257 )
7258 .unwrap();
7259
7260 let snapshot = wait_for_terminal_snapshot(
7261 ®istry,
7262 &task_id,
7263 &session_id,
7264 dir.path(),
7265 dir.path(),
7266 );
7267 assert_eq!(
7268 snapshot.info.status,
7269 BgTaskStatus::Completed,
7270 "{name}: task should complete; snapshot={snapshot:?}"
7271 );
7272 assert_eq!(
7273 snapshot.exit_code,
7274 Some(0),
7275 "{name}: script should use the final command's exit code"
7276 );
7277
7278 let stdout = String::from_utf8(
7279 registry
7280 .read_artifact(&task_id, &session_id, TaskArtifact::Stdout)
7281 .expect("read validated stdout artifact"),
7282 )
7283 .expect("stdout is UTF-8");
7284 let lines: Vec<&str> = stdout.lines().collect();
7285 assert_eq!(
7286 lines, expected_lines,
7287 "{name}: raw stdout artifact must include every newline-separated command's output"
7288 );
7289 }
7290 }
7291
7292 #[test]
7293 fn recognizes_all_recovery_marker_forms() {
7294 assert!(is_recovery_marker(
7295 "[truncated output; full output: read \"/tmp/out\"]"
7296 ));
7297 assert!(is_recovery_marker(
7298 "[omitted output; see remaining: tail -n +42 \"/tmp/out\"]"
7299 ));
7300 assert!(is_recovery_marker(
7301 "[truncated output; full output unavailable]"
7302 ));
7303 assert!(is_recovery_marker(
7304 r#"[truncated 123 bytes from saved output prefix; retained output: read "/tmp/out"]"#
7305 ));
7306 }
7307
7308 #[test]
7309 fn recovery_marker_reports_disk_prefix_truncation_as_retained_output() {
7310 let recovery = RecoveryContext {
7311 dropped_by_class: BTreeMap::new(),
7312 had_inner_drop: false,
7313 offset_hint_eligible: false,
7314 offset_start_line: None,
7315 byte_truncated: false,
7316 disk_truncated_prefix_bytes: 4096,
7317 output_path: Some("/tmp/stdout".to_string()),
7318 stderr_path: None,
7319 include_stderr_path: false,
7320 artifact_access: ArtifactRecoveryAccess {
7321 task_id: "bash-test".to_string(),
7322 readable: true,
7323 },
7324 };
7325
7326 let marker = recovery_marker(&recovery).expect("disk truncation must emit marker");
7327
7328 assert!(marker.contains("truncated 4096 bytes from saved output prefix"));
7329 assert!(marker.contains(r#"retained output: read "/tmp/stdout""#));
7330 assert!(!marker.contains("full output: read"));
7331 }
7332
7333 #[test]
7334 fn killed_exit_marker_sets_nonzero_sentinel_exit_code() {
7335 let metadata = PersistedTask::starting(
7336 "task".to_string(),
7337 "session".to_string(),
7338 "cargo test".to_string(),
7339 PathBuf::from("/tmp"),
7340 None,
7341 None,
7342 true,
7343 true,
7344 );
7345
7346 let terminal = terminal_metadata_from_marker(metadata, ExitMarker::Killed, None);
7347
7348 assert_eq!(terminal.status, BgTaskStatus::Killed);
7349 assert_eq!(terminal.exit_code, Some(137));
7350 }
7351
7352 #[test]
7353 fn terminal_status_polls_use_cached_render_once_and_off_lock() {
7354 let registry = BgTaskRegistry::default();
7355 let dir = tempfile::tempdir().unwrap();
7356 let (_task_id, task) = insert_terminal_piped_task(
7357 ®istry,
7358 &dir,
7359 "custom-tool --verbose",
7360 &"stdout line\n".repeat(200_000),
7361 "",
7362 true,
7363 );
7364 let calls = Arc::new(AtomicUsize::new(0));
7365 let saw_unlocked_state = Arc::new(AtomicBool::new(false));
7366 let task_holder = Arc::new(Mutex::new(Some(Arc::clone(&task))));
7367 let calls_for_closure = Arc::clone(&calls);
7368 let unlocked_for_closure = Arc::clone(&saw_unlocked_state);
7369 let task_for_closure = Arc::clone(&task_holder);
7370 registry.set_compressor_with_exit_code(move |_command, output, _exit_code| {
7371 calls_for_closure.fetch_add(1, Ordering::SeqCst);
7372 if let Some(task) = task_for_closure.lock().unwrap().as_ref() {
7373 if task.state.try_lock().is_ok() {
7374 unlocked_for_closure.store(true, Ordering::SeqCst);
7375 }
7376 }
7377 CompressionResult::new(format!("compressed {} bytes", output.len()))
7378 });
7379
7380 let first = registry
7381 .status(
7382 &task.task_id,
7383 "session",
7384 None,
7385 Some(dir.path()),
7386 RUNNING_OUTPUT_PREVIEW_BYTES,
7387 )
7388 .unwrap();
7389 let second = registry
7390 .status(
7391 &task.task_id,
7392 "session",
7393 None,
7394 Some(dir.path()),
7395 RUNNING_OUTPUT_PREVIEW_BYTES,
7396 )
7397 .unwrap();
7398 let listed = registry.list(RUNNING_OUTPUT_PREVIEW_BYTES);
7399
7400 assert_eq!(
7401 calls.load(Ordering::SeqCst),
7402 1,
7403 "terminal render must be cached"
7404 );
7405 assert!(
7406 saw_unlocked_state.load(Ordering::SeqCst),
7407 "compressor must run after releasing the task state lock"
7408 );
7409 assert!(first.output_preview.starts_with("compressed "));
7410 assert_eq!(second.output_preview, first.output_preview);
7411 assert_eq!(listed[0].output_preview, first.output_preview);
7412 }
7413
7414 #[test]
7415 fn completion_preview_success_keeps_tail_only() {
7416 let registry = BgTaskRegistry::default();
7421 let dir = tempfile::tempdir().unwrap();
7422 let output = format!("HEAD-SIGNAL\n{}TAIL-SIGNAL\n", "middle\n".repeat(2_000));
7423 let (_task_id, task) =
7424 insert_terminal_piped_task(®istry, &dir, "cat big.log", &output, "", false);
7425
7426 registry.post_terminal_transition(&task, true).unwrap();
7427 let completions = registry.drain_completions_for_session(Some("session"));
7428 assert_eq!(completions.len(), 1);
7429 let preview = &completions[0].output_preview;
7430 assert!(preview.contains("TAIL-SIGNAL"), "preview was {preview:?}");
7431 assert!(!preview.contains("HEAD-SIGNAL"), "preview was {preview:?}");
7432 assert!(completions[0].output_truncated);
7433 }
7434
7435 #[test]
7436 fn completion_preview_failure_keeps_head_and_tail() {
7437 let registry = BgTaskRegistry::default();
7440 let dir = tempfile::tempdir().unwrap();
7441 let output = format!("HEAD-SIGNAL\n{}TAIL-SIGNAL\n", "middle\n".repeat(2_000));
7442 let task_id = random_slug();
7443 let paths = task_paths(dir.path(), "session", &task_id).unwrap();
7444 fs::create_dir_all(&paths.dir).unwrap();
7445 fs::write(&paths.stdout, &output).unwrap();
7446 fs::write(&paths.stderr, "").unwrap();
7447 let mut metadata = PersistedTask::starting(
7448 task_id.clone(),
7449 "session".to_string(),
7450 "cat big.log".to_string(),
7451 dir.path().to_path_buf(),
7452 Some(dir.path().to_path_buf()),
7453 Some(30_000),
7454 true,
7455 false,
7456 );
7457 metadata.mark_terminal(BgTaskStatus::Failed, Some(1), None);
7458 write_task(&paths.json, &metadata).unwrap();
7459 registry
7460 .insert_rehydrated_task(metadata, paths, true)
7461 .expect("insert terminal task");
7462 let task = registry.task_for_session(&task_id, "session").unwrap();
7463
7464 registry.post_terminal_transition(&task, true).unwrap();
7465 let completions = registry.drain_completions_for_session(Some("session"));
7466 assert_eq!(completions.len(), 1);
7467 let preview = &completions[0].output_preview;
7468 assert!(preview.contains("HEAD-SIGNAL"), "preview was {preview:?}");
7469 assert!(preview.contains("TAIL-SIGNAL"), "preview was {preview:?}");
7470 }
7471
7472 #[test]
7473 fn has_completions_for_session_matches_pending_delivery() {
7474 let registry = BgTaskRegistry::default();
7475 assert!(!registry.has_completions_for_session(Some("session")));
7476 assert!(!registry.has_completions_for_session(None));
7477
7478 let dir = tempfile::tempdir().unwrap();
7479 let (_task_id, task) =
7480 insert_terminal_piped_task(®istry, &dir, QUICK_SUCCESS_COMMAND, "done\n", "", false);
7481 registry.post_terminal_transition(&task, true).unwrap();
7482
7483 assert!(registry.has_completions_for_session(Some("session")));
7484 assert!(registry.has_completions_for_session(None));
7485 assert!(!registry.has_completions_for_session(Some("other-session")));
7486
7487 let completions = registry.drain_completions_for_session(Some("session"));
7488 assert_eq!(completions.len(), 1);
7489 assert_eq!(completions[0].task_id, task.task_id);
7490 }
7491
7492 #[test]
7493 fn completion_drain_redelivers_until_ack_marks_it_delivered() {
7494 let registry = BgTaskRegistry::default();
7495 let dir = tempfile::tempdir().unwrap();
7496 let (task_id, task) =
7497 insert_terminal_piped_task(®istry, &dir, QUICK_SUCCESS_COMMAND, "done\n", "", false);
7498 registry.post_terminal_transition(&task, true).unwrap();
7499
7500 let first = registry.drain_completions_for_session(Some("session"));
7501 let second = registry.drain_completions_for_session(Some("session"));
7502 assert_eq!(first.len(), 1);
7503 assert_eq!(second.len(), 1);
7504 assert_eq!(first[0].task_id, task_id);
7505 assert_eq!(second[0].task_id, task_id);
7506
7507 let resolved =
7508 resolve_task_layout(&session_tasks_dir(dir.path(), "session"), &task_id).unwrap();
7509 assert!(!read_task_at(&resolved).unwrap().completion_delivered);
7510
7511 assert_eq!(
7512 registry.ack_completions_for_session(Some("session"), std::slice::from_ref(&task_id)),
7513 vec![task_id.clone()]
7514 );
7515 assert!(registry
7516 .drain_completions_for_session(Some("session"))
7517 .is_empty());
7518 assert!(read_task_at(&resolved).unwrap().completion_delivered);
7519 }
7520
7521 #[test]
7522 fn structured_gh_json_survives_intact_and_ignores_stderr() {
7523 let registry = BgTaskRegistry::default();
7524 let dir = tempfile::tempdir().unwrap();
7525 let calls = Arc::new(AtomicUsize::new(0));
7526 let calls_for_closure = Arc::clone(&calls);
7527 registry.set_compressor_with_exit_code(move |_command, output, _exit_code| {
7528 calls_for_closure.fetch_add(1, Ordering::SeqCst);
7529 CompressionResult::new(output)
7530 });
7531 let (task_id, _task) = insert_terminal_piped_task(
7532 ®istry,
7533 &dir,
7534 "gh pr view 123 --json body",
7535 "{\"body\":\"hello\"}",
7536 "warning: stderr must not join json",
7537 true,
7538 );
7539
7540 let snapshot = registry
7541 .status(
7542 &task_id,
7543 "session",
7544 None,
7545 Some(dir.path()),
7546 RUNNING_OUTPUT_PREVIEW_BYTES,
7547 )
7548 .unwrap();
7549
7550 assert_eq!(snapshot.output_preview, "{\"body\":\"hello\"}");
7551 assert!(!snapshot.output_preview.contains("warning"));
7552 assert!(!snapshot.output_truncated);
7553 assert_eq!(
7554 calls.load(Ordering::SeqCst),
7555 0,
7556 "structured JSON bypasses compression"
7557 );
7558 }
7559
7560 #[test]
7561 fn registry_emits_single_recovery_marker_for_class_drops() {
7562 let registry = BgTaskRegistry::default();
7563 let dir = tempfile::tempdir().unwrap();
7564 registry.set_compressor_with_exit_code(move |_command, _output, _exit_code| {
7565 let mut dropped = BTreeMap::new();
7566 dropped.insert(DropClass::Error, 18);
7567 dropped.insert(DropClass::Warning, 6);
7568 CompressionResult::with_class_drops("kept diagnostic", dropped)
7569 });
7570 let (task_id, task) =
7571 insert_terminal_piped_task(®istry, &dir, "custom-tool", "raw", "", true);
7572
7573 let snapshot = registry
7574 .status(
7575 &task_id,
7576 "session",
7577 None,
7578 Some(dir.path()),
7579 RUNNING_OUTPUT_PREVIEW_BYTES,
7580 )
7581 .unwrap();
7582
7583 assert_eq!(snapshot.output_preview.matches("full output:").count(), 1);
7584 assert!(snapshot.output_preview.contains("+18 more errors"));
7585 assert!(snapshot.output_preview.contains("+6 more warnings"));
7586 assert!(snapshot
7587 .output_preview
7588 .contains(&format!("read \"{}\"", task.paths.stdout.display())));
7589 assert!(!snapshot.output_preview.contains("tail -n +"));
7590 assert!(snapshot.output_truncated);
7591 }
7592
7593 #[test]
7594 fn registry_marker_reports_semantic_and_byte_drops_once() {
7595 let registry = BgTaskRegistry::default();
7596 let dir = tempfile::tempdir().unwrap();
7597 registry.set_compressor_with_exit_code(move |_command, _output, _exit_code| {
7598 let mut dropped = BTreeMap::new();
7599 dropped.insert(DropClass::Error, 1);
7600 CompressionResult::with_class_drops(
7601 format!("HEAD-SIGNAL\n{}TAIL-SIGNAL", "middle\n".repeat(8_000)),
7602 dropped,
7603 )
7604 });
7605 let (task_id, _task) =
7606 insert_terminal_piped_task(®istry, &dir, "custom-tool", "raw", "", true);
7607
7608 let snapshot = registry
7609 .status(
7610 &task_id,
7611 "session",
7612 None,
7613 Some(dir.path()),
7614 RUNNING_OUTPUT_PREVIEW_BYTES,
7615 )
7616 .unwrap();
7617
7618 assert_eq!(snapshot.output_preview.matches("full output:").count(), 1);
7619 assert!(snapshot.output_preview.contains("+1 more error"));
7620 assert!(snapshot.output_preview.contains("truncated output"));
7621 assert!(snapshot.output_preview.contains("HEAD-SIGNAL"));
7622 assert!(snapshot.output_preview.contains("TAIL-SIGNAL"));
7623 assert!(!snapshot.output_preview.contains("...<truncated"));
7624 assert!(snapshot.output_truncated);
7625 }
7626
7627 #[test]
7628 fn cargo_stderr_class_drops_name_both_capture_paths() {
7629 let registry = BgTaskRegistry::default();
7630 let dir = tempfile::tempdir().unwrap();
7631 let filter_registry = crate::compress::toml_filter::FilterRegistry::default();
7632 registry.set_compressor_with_exit_code(move |command, output, exit_code| {
7633 crate::compress::compress_with_registry_exit_code(
7634 command,
7635 &output,
7636 exit_code,
7637 &filter_registry,
7638 )
7639 });
7640 let stderr = (0..22)
7641 .map(|index| {
7642 format!(
7643 "error: cargo failure {index}\n --> src/lib.rs:{}:1\n |\n{} | boom\n",
7644 index + 1,
7645 index + 1
7646 )
7647 })
7648 .collect::<Vec<_>>()
7649 .join("\n");
7650 let (task_id, task) = insert_terminal_piped_task(
7651 ®istry,
7652 &dir,
7653 "cargo check",
7654 "Finished dev [unoptimized] target(s) in 0.01s\n",
7655 &stderr,
7656 true,
7657 );
7658
7659 let snapshot = registry
7660 .status(
7661 &task_id,
7662 "session",
7663 None,
7664 Some(dir.path()),
7665 RUNNING_OUTPUT_PREVIEW_BYTES,
7666 )
7667 .unwrap();
7668
7669 assert!(snapshot.output_preview.contains("+2 more errors"));
7670 assert!(snapshot
7671 .output_preview
7672 .contains(&format!("read \"{}\"", task.paths.stdout.display())));
7673 assert!(snapshot
7674 .output_preview
7675 .contains(&format!("read \"{}\"", task.paths.stderr.display())));
7676 assert!(!snapshot.output_preview.contains("tail -n +"));
7677 }
7678
7679 #[test]
7680 fn over_ceiling_structured_json_uses_pointer_not_partial_json() {
7681 let registry = BgTaskRegistry::default();
7682 let dir = tempfile::tempdir().unwrap();
7683 let body = format!("{{\"body\":\"{}\"}}", "x".repeat(60 * 1024));
7684 let (task_id, task) = insert_terminal_piped_task(
7685 ®istry,
7686 &dir,
7687 "cd /repo && gh pr view 123 --json body",
7688 &body,
7689 "",
7690 true,
7691 );
7692
7693 let snapshot = registry
7694 .status(
7695 &task_id,
7696 "session",
7697 None,
7698 Some(dir.path()),
7699 RUNNING_OUTPUT_PREVIEW_BYTES,
7700 )
7701 .unwrap();
7702
7703 assert!(snapshot.output_preview.starts_with("[JSON output "));
7704 assert!(snapshot
7705 .output_preview
7706 .contains(&task.paths.stdout.display().to_string()));
7707 assert!(!snapshot.output_preview.contains(&"x".repeat(1024)));
7708 assert!(snapshot.output_truncated);
7709 }
7710
7711 #[test]
7712 fn toml_strip_tail_cap_uses_full_output_hint_not_offset_hint() {
7713 let registry = BgTaskRegistry::default();
7714 let dir = tempfile::tempdir().unwrap();
7715 let filter_registry = crate::compress::toml_filter::build_registry(
7716 crate::compress::builtin_filters::ALL,
7717 None,
7718 None,
7719 );
7720 registry.set_compressor_with_exit_code(move |command, output, exit_code| {
7721 crate::compress::compress_with_registry_exit_code(
7722 command,
7723 &output,
7724 exit_code,
7725 &filter_registry,
7726 )
7727 });
7728 let stdout = format!(
7729 "make[1]: Entering directory `/tmp`\n{}",
7730 (0..100)
7731 .map(|index| format!("compile line {index}"))
7732 .collect::<Vec<_>>()
7733 .join("\n")
7734 );
7735 let (task_id, task) =
7736 insert_terminal_piped_task(®istry, &dir, "make all", &stdout, "", true);
7737
7738 let snapshot = registry
7739 .status(
7740 &task_id,
7741 "session",
7742 None,
7743 Some(dir.path()),
7744 RUNNING_OUTPUT_PREVIEW_BYTES,
7745 )
7746 .unwrap();
7747
7748 assert!(snapshot.output_preview.contains("compile line 99"));
7749 assert!(snapshot.output_preview.contains(&format!(
7750 "full output: read \"{}\"",
7751 task.paths.stdout.display()
7752 )));
7753 assert!(!snapshot
7754 .output_preview
7755 .contains(&format!("read \"{}\"", task.paths.stderr.display())));
7756 assert!(!snapshot.output_preview.contains("tail -n +"));
7757 }
7758
7759 #[test]
7760 fn compressed_false_raw_passthrough_uses_wider_head_tail_cap() {
7761 let registry = BgTaskRegistry::default();
7762 let dir = tempfile::tempdir().unwrap();
7763 let output = format!("RAW-HEAD\n{}RAW-TAIL\n", "raw-middle\n".repeat(8_000));
7764 let (task_id, task) =
7765 insert_terminal_piped_task(®istry, &dir, "cat raw.log", &output, "RAW-ERR\n", false);
7766
7767 let snapshot = registry
7768 .status(
7769 &task_id,
7770 "session",
7771 None,
7772 Some(dir.path()),
7773 RUNNING_OUTPUT_PREVIEW_BYTES,
7774 )
7775 .unwrap();
7776
7777 assert!(snapshot.output_preview.contains("RAW-HEAD"));
7778 assert!(snapshot.output_preview.contains("RAW-TAIL"));
7779 assert!(snapshot.output_preview.contains("truncated output"));
7780 assert!(snapshot
7781 .output_preview
7782 .contains(&format!("read \"{}\"", task.paths.stdout.display())));
7783 assert!(snapshot
7784 .output_preview
7785 .contains(&format!("read \"{}\"", task.paths.stderr.display())));
7786 assert!(!snapshot.output_preview.contains("tail -n +"));
7787 assert!(snapshot.output_preview.len() > 16 * 1024);
7788 assert!(snapshot.output_truncated);
7789 }
7790
7791 #[test]
7792 fn pty_terminal_snapshot_bypasses_line_compression() {
7793 let registry = BgTaskRegistry::default();
7794 let dir = tempfile::tempdir().unwrap();
7795 let calls = Arc::new(AtomicUsize::new(0));
7796 let calls_for_closure = Arc::clone(&calls);
7797 registry.set_compressor_with_exit_code(move |_command, output, _exit_code| {
7798 calls_for_closure.fetch_add(1, Ordering::SeqCst);
7799 CompressionResult::new(output)
7800 });
7801 let (task_id, _task) = insert_terminal_pty_task(®istry, &dir, "raw\u{1b}[31m pty bytes");
7802
7803 let snapshot = registry
7804 .status(
7805 &task_id,
7806 "session",
7807 None,
7808 Some(dir.path()),
7809 RUNNING_OUTPUT_PREVIEW_BYTES,
7810 )
7811 .unwrap();
7812
7813 assert_eq!(snapshot.info.mode, BgMode::Pty);
7814 assert_eq!(snapshot.output_preview, "");
7815 assert_eq!(calls.load(Ordering::SeqCst), 0);
7816 }
7817
7818 #[test]
7819 fn pty_dimensions_are_persisted_and_returned_in_snapshot() {
7820 let registry = BgTaskRegistry::default();
7821 let dir = tempfile::tempdir().unwrap();
7822 let task_id = registry
7823 .spawn_pty(
7824 SpawnPlan::Unsandboxed,
7825 QUICK_SUCCESS_COMMAND,
7826 "session".to_string(),
7827 dir.path().to_path_buf(),
7828 HashMap::new(),
7829 Some(Duration::from_secs(30)),
7830 dir.path().to_path_buf(),
7831 10,
7832 true,
7833 false,
7834 Some(dir.path().to_path_buf()),
7835 50,
7836 120,
7837 )
7838 .unwrap();
7839
7840 let resolved =
7841 resolve_task_layout(&session_tasks_dir(dir.path(), "session"), &task_id).unwrap();
7842 let metadata = read_task_at(&resolved).unwrap();
7843 assert_eq!(
7844 metadata.schema_version,
7845 crate::bash_background::persistence::SCHEMA_VERSION
7846 );
7847 assert_eq!(metadata.mode, BgMode::Pty);
7848 assert_eq!(metadata.pty_rows, Some(50));
7849 assert_eq!(metadata.pty_cols, Some(120));
7850
7851 let snapshot = registry
7852 .status(&task_id, "session", None, Some(dir.path()), 1024)
7853 .unwrap();
7854 assert_eq!(snapshot.pty_rows, Some(50));
7855 assert_eq!(snapshot.pty_cols, Some(120));
7856 }
7857
7858 fn spawn_dead_child() -> std::process::Child {
7863 #[cfg(unix)]
7864 let mut cmd = std::process::Command::new("true");
7865 #[cfg(windows)]
7866 let mut cmd = {
7867 let mut c = std::process::Command::new("cmd");
7868 c.args(["/c", "exit", "0"]);
7869 c
7870 };
7871 cmd.stdin(std::process::Stdio::null());
7872 cmd.stdout(std::process::Stdio::null());
7873 cmd.stderr(std::process::Stdio::null());
7874 let mut child = cmd.spawn().expect("spawn replacement child for reap test");
7875 let started = Instant::now();
7884 loop {
7885 match child.try_wait() {
7886 Ok(Some(_)) => break,
7887 Ok(None) => {
7888 if started.elapsed() > CHILD_EXIT_LIVENESS_BOUND {
7889 panic!("dead-child stand-in did not exit within the liveness bound");
7890 }
7891 std::thread::sleep(Duration::from_millis(10));
7892 }
7893 Err(error) => panic!("dead-child try_wait failed: {error}"),
7894 }
7895 }
7896 child
7897 }
7898
7899 #[test]
7900 fn ack_marks_delivered_even_when_completion_was_already_consumed_locally() {
7901 let registry = BgTaskRegistry::default();
7902 let dir = tempfile::tempdir().unwrap();
7903 let task_id = registry
7904 .spawn(
7905 SpawnPlan::Unsandboxed,
7906 LONG_RUNNING_COMMAND,
7907 "session".to_string(),
7908 dir.path().to_path_buf(),
7909 HashMap::new(),
7910 Some(Duration::from_secs(30)),
7911 dir.path().to_path_buf(),
7912 10,
7913 true,
7914 false,
7915 Some(dir.path().to_path_buf()),
7916 )
7917 .unwrap();
7918 registry
7919 .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
7920 .unwrap();
7921 assert_eq!(
7922 registry
7923 .drain_completions_for_session(Some("session"))
7924 .len(),
7925 1
7926 );
7927
7928 registry.inner.completions.lock().unwrap().clear();
7931
7932 assert_eq!(
7933 registry.ack_completions_for_session(Some("session"), std::slice::from_ref(&task_id)),
7934 vec![task_id.clone()]
7935 );
7936 assert!(registry
7937 .drain_completions_for_session(Some("session"))
7938 .is_empty());
7939
7940 let resolved =
7941 resolve_task_layout(&session_tasks_dir(dir.path(), "session"), &task_id).unwrap();
7942 let metadata = read_task_at(&resolved).unwrap();
7943 assert!(metadata.completion_delivered);
7944
7945 let replayed = BgTaskRegistry::default();
7946 replayed
7947 .replay_session_inner(dir.path(), "session", None)
7948 .unwrap();
7949 assert!(replayed
7950 .drain_completions_for_session(Some("session"))
7951 .is_empty());
7952 }
7953
7954 #[test]
7955 fn reclaimed_root_kills_running_task_and_persists_reason() {
7956 let registry = BgTaskRegistry::default();
7957 let root = tempfile::tempdir().unwrap();
7958 let storage = tempfile::tempdir().unwrap();
7959 let task_id = registry
7960 .spawn(
7961 SpawnPlan::Unsandboxed,
7962 LONG_RUNNING_COMMAND,
7963 "session".to_string(),
7964 root.path().to_path_buf(),
7965 HashMap::new(),
7966 Some(Duration::from_secs(30)),
7967 storage.path().to_path_buf(),
7968 10,
7969 true,
7970 false,
7971 Some(root.path().to_path_buf()),
7972 )
7973 .unwrap();
7974 let pid = registry
7975 .status(
7976 &task_id,
7977 "session",
7978 Some(root.path()),
7979 Some(storage.path()),
7980 0,
7981 )
7982 .unwrap()
7983 .child_pid
7984 .unwrap();
7985 assert!(is_process_alive(pid));
7986
7987 assert_eq!(registry.kill_running_tasks_for_root(root.path()), 1);
7988 let deadline = Instant::now() + CHILD_EXIT_LIVENESS_BOUND;
7989 while is_process_alive(pid) {
7990 assert!(
7991 Instant::now() < deadline,
7992 "reclaimed task process survived kill"
7993 );
7994 std::thread::sleep(Duration::from_millis(20));
7995 }
7996
7997 let snapshot = registry
7998 .status(
7999 &task_id,
8000 "session",
8001 Some(root.path()),
8002 Some(storage.path()),
8003 0,
8004 )
8005 .unwrap();
8006 assert_eq!(snapshot.info.status, BgTaskStatus::Killed);
8007 assert_eq!(
8008 snapshot.info.status_reason.as_deref(),
8009 Some(ROOT_RECLAIMED_REASON)
8010 );
8011 let persisted = read_task(
8012 ®istry
8013 .task_json_path(&task_id, "session")
8014 .expect("reclaimed task metadata path"),
8015 )
8016 .expect("persisted reclaimed task");
8017 assert_eq!(
8018 persisted.status_reason.as_deref(),
8019 Some(ROOT_RECLAIMED_REASON)
8020 );
8021 let completion = registry
8022 .drain_completions_for_session(Some("session"))
8023 .pop()
8024 .expect("reclaimed task completion");
8025 assert_eq!(
8026 completion.status_reason.as_deref(),
8027 Some(ROOT_RECLAIMED_REASON)
8028 );
8029 registry.detach();
8030 }
8031
8032 #[test]
8033 fn reclaimed_root_kills_pty_task_and_preserves_reason() {
8034 let registry = BgTaskRegistry::default();
8035 let root = tempfile::tempdir().unwrap();
8036 let storage = tempfile::tempdir().unwrap();
8037 let command = if cfg!(windows) {
8038 "Start-Sleep -Seconds 30"
8039 } else {
8040 "sleep 30"
8041 };
8042 let task_id = registry
8043 .spawn_pty(
8044 SpawnPlan::Unsandboxed,
8045 command,
8046 "session".to_string(),
8047 root.path().to_path_buf(),
8048 HashMap::new(),
8049 Some(Duration::from_secs(60)),
8050 storage.path().to_path_buf(),
8051 10,
8052 true,
8053 false,
8054 Some(root.path().to_path_buf()),
8055 24,
8056 80,
8057 )
8058 .unwrap();
8059 let pid = registry
8060 .status(
8061 &task_id,
8062 "session",
8063 Some(root.path()),
8064 Some(storage.path()),
8065 0,
8066 )
8067 .unwrap()
8068 .child_pid
8069 .unwrap();
8070 assert!(is_process_alive(pid));
8071
8072 assert_eq!(registry.kill_running_tasks_for_root(root.path()), 1);
8073 let deadline = Instant::now() + Duration::from_secs(10);
8074 loop {
8075 let snapshot = registry
8076 .status(
8077 &task_id,
8078 "session",
8079 Some(root.path()),
8080 Some(storage.path()),
8081 0,
8082 )
8083 .unwrap();
8084 if snapshot.info.status.is_terminal() {
8085 assert_eq!(snapshot.info.status, BgTaskStatus::Killed);
8086 assert_eq!(
8087 snapshot.info.status_reason.as_deref(),
8088 Some(ROOT_RECLAIMED_REASON)
8089 );
8090 break;
8091 }
8092 assert!(
8093 Instant::now() < deadline,
8094 "reclaimed PTY task did not terminate"
8095 );
8096 std::thread::sleep(Duration::from_millis(20));
8097 }
8098 assert!(!is_process_alive(pid));
8099 let completion = loop {
8104 if let Some(completion) = registry
8105 .drain_completions_for_session(Some("session"))
8106 .pop()
8107 {
8108 break completion;
8109 }
8110 assert!(
8111 Instant::now() < deadline,
8112 "reclaimed PTY completion never arrived"
8113 );
8114 std::thread::sleep(Duration::from_millis(20));
8115 };
8116 assert_eq!(
8117 completion.status_reason.as_deref(),
8118 Some(ROOT_RECLAIMED_REASON)
8119 );
8120 registry.detach();
8121 }
8122
8123 #[test]
8124 fn register_watch_rejects_unknown_task() {
8125 let registry = BgTaskRegistry::default();
8126
8127 let result = registry.register_watch(
8128 "missing-task".to_string(),
8129 WatchPattern::Substring("READY".into()),
8130 true,
8131 );
8132
8133 assert_eq!(result, Err("task_not_found"));
8134 }
8135
8136 #[test]
8137 fn register_watch_on_terminal_task_scans_existing_output() {
8138 let frames = Arc::new(Mutex::new(Vec::new()));
8139 let captured = Arc::clone(&frames);
8140 let sender: crate::context::ProgressSender = Arc::new(Box::new(move |frame| {
8141 captured.lock().unwrap().push(frame);
8142 })
8143 as Box<dyn Fn(PushFrame) + Send + Sync>);
8144 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(Some(sender))));
8145 let dir = tempfile::tempdir().unwrap();
8146 let task_id = registry
8147 .spawn(
8148 SpawnPlan::Unsandboxed,
8149 LONG_RUNNING_COMMAND,
8150 "session".to_string(),
8151 dir.path().to_path_buf(),
8152 HashMap::new(),
8153 Some(Duration::from_secs(30)),
8154 dir.path().to_path_buf(),
8155 10,
8156 true,
8157 false,
8158 Some(dir.path().to_path_buf()),
8159 )
8160 .unwrap();
8161 registry
8162 .inner
8163 .shutdown
8164 .store(true, std::sync::atomic::Ordering::SeqCst);
8165 let task = registry.task_for_session(&task_id, "session").unwrap();
8166 std::fs::write(&task.paths.stdout, "READY\n").unwrap();
8167 registry
8168 .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
8169 .unwrap();
8170 frames.lock().unwrap().clear();
8171 registry.inner.completions.lock().unwrap().clear();
8172
8173 registry
8174 .register_watch(
8175 task_id.clone(),
8176 WatchPattern::Substring("READY".into()),
8177 true,
8178 )
8179 .unwrap();
8180
8181 let frames = frames.lock().unwrap();
8182 let frame = frames
8183 .iter()
8184 .find_map(|frame| match frame {
8185 PushFrame::BashPatternMatch(frame) => Some(frame),
8186 _ => None,
8187 })
8188 .expect("terminal watch registration should emit pattern frame");
8189 assert_eq!(frame.reason, "pattern_match");
8190 assert_eq!(frame.task_id, task_id);
8191 assert_eq!(frame.session_id, "session");
8192 assert_eq!(frame.match_text, "READY");
8193 assert_eq!(frame.match_offset, 0);
8194 assert_eq!(registry.active_watch_count(&frame.task_id), 0);
8195 let metadata = read_task(&task.paths.json).unwrap();
8196 assert!(
8197 !metadata.completion_delivered,
8198 "terminal pattern notification remains unacked until explicit ack"
8199 );
8200 drop(frames);
8201 assert_eq!(
8202 registry.ack_completions_for_session(Some("session"), std::slice::from_ref(&task_id),),
8203 vec![task_id]
8204 );
8205 assert!(read_task(&task.paths.json).unwrap().completion_delivered);
8206 }
8207
8208 #[test]
8209 fn cleanup_finished_removes_terminal_tasks_older_than_threshold() {
8210 let registry = BgTaskRegistry::default();
8211 let dir = tempfile::tempdir().unwrap();
8212 let task_id = registry
8213 .spawn(
8214 SpawnPlan::Unsandboxed,
8215 QUICK_SUCCESS_COMMAND,
8216 "session".to_string(),
8217 dir.path().to_path_buf(),
8218 HashMap::new(),
8219 Some(Duration::from_secs(30)),
8220 dir.path().to_path_buf(),
8221 10,
8222 true,
8223 false,
8224 Some(dir.path().to_path_buf()),
8225 )
8226 .unwrap();
8227 registry
8228 .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
8229 .unwrap();
8230 let completions = registry.drain_completions_for_session(Some("session"));
8231 assert_eq!(completions.len(), 1);
8232 assert_eq!(
8233 registry.ack_completions_for_session(Some("session"), std::slice::from_ref(&task_id)),
8234 vec![task_id.clone()]
8235 );
8236
8237 registry.cleanup_finished(Duration::ZERO);
8238
8239 assert!(registry.inner.tasks.lock().unwrap().is_empty());
8240 }
8241
8242 #[test]
8243 fn cleanup_finished_retains_undelivered_terminals() {
8244 let registry = BgTaskRegistry::default();
8245 let dir = tempfile::tempdir().unwrap();
8246 let task_id = registry
8247 .spawn(
8248 SpawnPlan::Unsandboxed,
8249 QUICK_SUCCESS_COMMAND,
8250 "session".to_string(),
8251 dir.path().to_path_buf(),
8252 HashMap::new(),
8253 Some(Duration::from_secs(30)),
8254 dir.path().to_path_buf(),
8255 10,
8256 true,
8257 false,
8258 Some(dir.path().to_path_buf()),
8259 )
8260 .unwrap();
8261 registry
8262 .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
8263 .unwrap();
8264
8265 registry.cleanup_finished(Duration::ZERO);
8266
8267 assert!(registry.inner.tasks.lock().unwrap().contains_key(&task_id));
8268 }
8269
8270 #[test]
8278 fn reap_child_marks_failed_when_child_exits_without_exit_marker() {
8279 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
8280 let dir = tempfile::tempdir().unwrap();
8281 let task_id = registry
8282 .spawn(
8283 SpawnPlan::Unsandboxed,
8284 QUICK_SUCCESS_COMMAND,
8285 "session".to_string(),
8286 dir.path().to_path_buf(),
8287 HashMap::new(),
8288 Some(Duration::from_secs(30)),
8289 dir.path().to_path_buf(),
8290 10,
8291 true,
8292 false,
8293 Some(dir.path().to_path_buf()),
8294 )
8295 .unwrap();
8296
8297 let task = registry.task_for_session(&task_id, "session").unwrap();
8298
8299 let started = Instant::now();
8304 loop {
8305 let exited = {
8306 let mut state = task.state.lock().unwrap();
8307 match &mut state.runtime {
8308 TaskRuntime::Piped(Some(child)) => matches!(child.try_wait(), Ok(Some(_))),
8309 _ => true,
8310 }
8311 };
8312 if exited {
8313 break;
8314 }
8315 assert!(
8316 started.elapsed() < CHILD_EXIT_LIVENESS_BOUND,
8317 "child should exit quickly"
8318 );
8319 std::thread::sleep(Duration::from_millis(20));
8320 }
8321
8322 registry
8330 .inner
8331 .shutdown
8332 .store(true, std::sync::atomic::Ordering::SeqCst);
8333 std::thread::sleep(Duration::from_millis(550));
8337
8338 let _ = std::fs::remove_file(&task.paths.exit);
8341
8342 {
8357 let mut state = task.state.lock().unwrap();
8358 state.metadata.status = BgTaskStatus::Running;
8359 state.metadata.status_reason = None;
8360 state.metadata.exit_code = None;
8361 state.metadata.finished_at = None;
8362 state.metadata.duration_ms = None;
8363 crate::bash_background::persistence::write_task(&task.paths.json, &state.metadata)
8366 .expect("persist reset Running metadata for reap_child test");
8367 if matches!(state.runtime, TaskRuntime::Piped(None)) {
8371 state.runtime = TaskRuntime::Piped(Some(spawn_dead_child()));
8372 }
8373 }
8374 *task.terminal_at.lock().unwrap() = None;
8377
8378 assert!(
8381 task.is_running(),
8382 "precondition: metadata.status == Running"
8383 );
8384 assert!(
8385 !task.paths.exit.exists(),
8386 "precondition: exit marker absent"
8387 );
8388
8389 registry.reap_child(&task);
8394
8395 {
8396 let state = task.state.lock().unwrap();
8397 assert_eq!(
8398 state.metadata.status,
8399 BgTaskStatus::Running,
8400 "first reap must leave status Running while waiting one pass for marker"
8401 );
8402 assert_eq!(
8403 state.metadata.status_reason, None,
8404 "first reap must not record a failure reason"
8405 );
8406 assert!(
8407 matches!(state.runtime, TaskRuntime::Piped(None)),
8408 "child handle must be released after first reap"
8409 );
8410 assert!(
8411 state.detached,
8412 "task must be marked detached after first reap"
8413 );
8414 }
8415
8416 registry.reap_child(&task);
8420
8421 let state = task.state.lock().unwrap();
8422 assert!(
8423 state.metadata.status.is_terminal(),
8424 "second reap must transition to terminal when PID dead and no marker. Got status={:?}",
8425 state.metadata.status
8426 );
8427 assert_eq!(
8428 state.metadata.status,
8429 BgTaskStatus::Failed,
8430 "must specifically be Failed (not Killed): status={:?}",
8431 state.metadata.status
8432 );
8433 assert_eq!(
8434 state.metadata.status_reason.as_deref(),
8435 Some("process exited without exit marker"),
8436 "reason must match replay path's wording: {:?}",
8437 state.metadata.status_reason
8438 );
8439 assert!(
8440 matches!(state.runtime, TaskRuntime::Piped(None)),
8441 "child handle must stay released after second reap"
8442 );
8443 assert!(
8444 state.detached,
8445 "task must remain detached after second reap"
8446 );
8447 }
8448
8449 #[test]
8454 fn reap_child_preserves_running_when_exit_marker_exists() {
8455 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
8456 let dir = tempfile::tempdir().unwrap();
8457 let task_id = registry
8458 .spawn(
8459 SpawnPlan::Unsandboxed,
8460 QUICK_SUCCESS_COMMAND,
8461 "session".to_string(),
8462 dir.path().to_path_buf(),
8463 HashMap::new(),
8464 Some(Duration::from_secs(30)),
8465 dir.path().to_path_buf(),
8466 10,
8467 true,
8468 false,
8469 Some(dir.path().to_path_buf()),
8470 )
8471 .unwrap();
8472
8473 let task = registry.task_for_session(&task_id, "session").unwrap();
8474
8475 let started = Instant::now();
8478 loop {
8479 let exited = {
8480 let mut state = task.state.lock().unwrap();
8481 match &mut state.runtime {
8482 TaskRuntime::Piped(Some(child)) => matches!(child.try_wait(), Ok(Some(_))),
8483 _ => true,
8484 }
8485 };
8486 if exited && task.paths.exit.exists() {
8487 break;
8488 }
8489 assert!(
8490 started.elapsed() < CHILD_EXIT_LIVENESS_BOUND,
8491 "child should exit and write marker quickly"
8492 );
8493 std::thread::sleep(Duration::from_millis(20));
8494 }
8495
8496 registry
8502 .inner
8503 .shutdown
8504 .store(true, std::sync::atomic::Ordering::SeqCst);
8505 std::thread::sleep(Duration::from_millis(550));
8506
8507 {
8513 let mut state = task.state.lock().unwrap();
8514 state.metadata.status = BgTaskStatus::Running;
8515 state.metadata.status_reason = None;
8516 if matches!(state.runtime, TaskRuntime::Piped(None)) {
8517 state.runtime = TaskRuntime::Piped(Some(spawn_dead_child()));
8518 }
8519 }
8520 *task.terminal_at.lock().unwrap() = None;
8521 if !task.paths.exit.exists() {
8524 std::fs::write(&task.paths.exit, "0").expect("write replacement exit marker");
8525 }
8526
8527 registry.reap_child(&task);
8531
8532 let state = task.state.lock().unwrap();
8533 assert!(
8534 matches!(state.runtime, TaskRuntime::Piped(None)),
8535 "child handle still released even when marker exists"
8536 );
8537 assert!(
8538 state.detached,
8539 "task still marked detached even when marker exists"
8540 );
8541 assert_eq!(
8546 state.metadata.status,
8547 BgTaskStatus::Running,
8548 "reap_child must defer to poll_task when marker exists"
8549 );
8550 }
8551
8552 #[cfg(unix)]
8556 fn pid_stat(pid: u32) -> Option<String> {
8557 let output = std::process::Command::new("ps")
8558 .args(["-o", "stat=", "-p", &pid.to_string()])
8559 .output()
8560 .ok()?;
8561 if !output.status.success() {
8562 return None;
8563 }
8564 let stat = String::from_utf8_lossy(&output.stdout).trim().to_string();
8565 if stat.is_empty() {
8566 None
8567 } else {
8568 Some(stat)
8569 }
8570 }
8571
8572 #[cfg(unix)]
8574 fn is_zombie(pid: u32) -> bool {
8575 pid_stat(pid).is_some_and(|stat| stat.starts_with('Z'))
8576 }
8577
8578 #[cfg(unix)]
8584 fn spawn_unreaped_zombie() -> std::process::Child {
8585 let child = std::process::Command::new("true")
8586 .stdin(std::process::Stdio::null())
8587 .stdout(std::process::Stdio::null())
8588 .stderr(std::process::Stdio::null())
8589 .spawn()
8590 .expect("spawn zombie stand-in");
8591 let pid = child.id();
8592 let started = Instant::now();
8593 while !is_zombie(pid) {
8594 assert!(
8595 started.elapsed() < CHILD_EXIT_LIVENESS_BOUND,
8596 "stand-in child should become a zombie within the liveness bound"
8597 );
8598 std::thread::sleep(Duration::from_millis(10));
8599 }
8600 child
8602 }
8603
8604 #[cfg(unix)]
8614 #[test]
8615 fn finalize_from_marker_reaps_child_no_zombie() {
8616 use std::sync::atomic::Ordering;
8617
8618 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
8619 let dir = tempfile::tempdir().unwrap();
8620 let task_id = registry
8621 .spawn(
8622 SpawnPlan::Unsandboxed,
8623 QUICK_SUCCESS_COMMAND,
8624 "session".to_string(),
8625 dir.path().to_path_buf(),
8626 HashMap::new(),
8627 Some(Duration::from_secs(30)),
8628 dir.path().to_path_buf(),
8629 10,
8630 true,
8631 false,
8632 Some(dir.path().to_path_buf()),
8633 )
8634 .unwrap();
8635
8636 registry.inner.shutdown.store(true, Ordering::SeqCst);
8640 std::thread::sleep(Duration::from_millis(550));
8641
8642 let task = registry.task_for_session(&task_id, "session").unwrap();
8643
8644 let started = Instant::now();
8648 while !task.paths.exit.exists() {
8649 assert!(
8650 started.elapsed() < CHILD_EXIT_LIVENESS_BOUND,
8651 "exit marker should land quickly for `true`"
8652 );
8653 std::thread::sleep(Duration::from_millis(20));
8654 }
8655
8656 let zombie_pid;
8662 {
8663 let mut state = task.state.lock().unwrap();
8664 state.metadata.status = BgTaskStatus::Running;
8665 state.metadata.status_reason = None;
8666 state.metadata.exit_code = None;
8667 state.metadata.finished_at = None;
8668 state.metadata.duration_ms = None;
8669 crate::bash_background::persistence::write_task(&task.paths.json, &state.metadata)
8670 .expect("persist reset Running metadata");
8671 let zombie = spawn_unreaped_zombie();
8672 zombie_pid = zombie.id();
8673 state.runtime = TaskRuntime::Piped(Some(zombie));
8674 }
8675 *task.terminal_at.lock().unwrap() = None;
8676
8677 assert!(
8679 is_zombie(zombie_pid),
8680 "precondition: stand-in child {zombie_pid} must be a zombie before finalize"
8681 );
8682
8683 registry.poll_task(&task).unwrap();
8686
8687 {
8688 let state = task.state.lock().unwrap();
8689 assert!(
8690 matches!(state.runtime, TaskRuntime::Piped(None)),
8691 "child handle must be released after marker finalize"
8692 );
8693 assert!(
8694 state.metadata.status.is_terminal(),
8695 "task must be terminal after marker finalize: {:?}",
8696 state.metadata.status
8697 );
8698 }
8699
8700 assert!(
8703 !is_zombie(zombie_pid),
8704 "issue #91 regression: child {zombie_pid} left as <defunct> zombie \
8705 after the exit-marker terminal transition"
8706 );
8707 }
8708
8709 #[cfg(unix)]
8713 #[test]
8714 fn kill_with_existing_marker_reaps_child_no_zombie() {
8715 use std::sync::atomic::Ordering;
8716
8717 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
8718 let dir = tempfile::tempdir().unwrap();
8719 let task_id = registry
8720 .spawn(
8721 SpawnPlan::Unsandboxed,
8722 QUICK_SUCCESS_COMMAND,
8723 "session".to_string(),
8724 dir.path().to_path_buf(),
8725 HashMap::new(),
8726 Some(Duration::from_secs(30)),
8727 dir.path().to_path_buf(),
8728 10,
8729 true,
8730 false,
8731 Some(dir.path().to_path_buf()),
8732 )
8733 .unwrap();
8734
8735 registry.inner.shutdown.store(true, Ordering::SeqCst);
8736 std::thread::sleep(Duration::from_millis(550));
8737
8738 let task = registry.task_for_session(&task_id, "session").unwrap();
8739
8740 let started = Instant::now();
8741 while !task.paths.exit.exists() {
8742 assert!(
8743 started.elapsed() < CHILD_EXIT_LIVENESS_BOUND,
8744 "exit marker should land quickly for `true`"
8745 );
8746 std::thread::sleep(Duration::from_millis(20));
8747 }
8748
8749 let zombie_pid;
8750 {
8751 let mut state = task.state.lock().unwrap();
8752 state.metadata.status = BgTaskStatus::Running;
8753 state.metadata.status_reason = None;
8754 state.metadata.exit_code = None;
8755 state.metadata.finished_at = None;
8756 state.metadata.duration_ms = None;
8757 crate::bash_background::persistence::write_task(&task.paths.json, &state.metadata)
8758 .expect("persist reset Running metadata");
8759 let zombie = spawn_unreaped_zombie();
8760 zombie_pid = zombie.id();
8761 state.runtime = TaskRuntime::Piped(Some(zombie));
8762 }
8763 *task.terminal_at.lock().unwrap() = None;
8764
8765 assert!(
8766 is_zombie(zombie_pid),
8767 "precondition: stand-in child {zombie_pid} must be a zombie before kill"
8768 );
8769
8770 registry
8772 .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
8773 .expect("kill should succeed");
8774
8775 {
8776 let state = task.state.lock().unwrap();
8777 assert!(
8778 matches!(state.runtime, TaskRuntime::Piped(None)),
8779 "child handle must be released after marker-aware kill"
8780 );
8781 assert!(state.metadata.status.is_terminal());
8782 }
8783
8784 assert!(
8785 !is_zombie(zombie_pid),
8786 "issue #91 regression: child {zombie_pid} left as <defunct> zombie \
8787 after a marker-aware kill"
8788 );
8789 }
8790
8791 #[test]
8792 fn cleanup_finished_keeps_running_tasks() {
8793 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
8794 let dir = tempfile::tempdir().unwrap();
8795 let task_id = registry
8796 .spawn(
8797 SpawnPlan::Unsandboxed,
8798 LONG_RUNNING_COMMAND,
8799 "session".to_string(),
8800 dir.path().to_path_buf(),
8801 HashMap::new(),
8802 Some(Duration::from_secs(30)),
8803 dir.path().to_path_buf(),
8804 10,
8805 true,
8806 false,
8807 Some(dir.path().to_path_buf()),
8808 )
8809 .unwrap();
8810
8811 registry.cleanup_finished(Duration::ZERO);
8812
8813 assert!(registry.inner.tasks.lock().unwrap().contains_key(&task_id));
8814 let _ = registry.kill(&task_id, "session");
8815 }
8816
8817 #[cfg(unix)]
8818 #[test]
8819 fn rehydrating_sandboxed_task_never_respawns_persisted_command() {
8820 let project = tempfile::tempdir().unwrap();
8821 let storage = tempfile::tempdir().unwrap();
8822 let sandbox_temp = storage.path().join("sandbox-temp");
8823 fs::create_dir(&sandbox_temp).unwrap();
8824 let launcher_script = project.path().join("sandbox-launch");
8825 let launcher = PathBuf::from("/bin/sh");
8826 fs::write(
8827 &launcher_script,
8828 "while [ \"$#\" -gt 0 ]; do\n if [ \"$1\" = -- ]; then\n shift\n exec \"$@\"\n fi\n shift\ndone\nexit 78\n",
8829 )
8830 .unwrap();
8831 let mut permissions = fs::metadata(&launcher_script).unwrap().permissions();
8832 permissions.set_mode(0o700);
8833 fs::set_permissions(&launcher_script, permissions).unwrap();
8834
8835 let profile = crate::sandbox_profile::SandboxProfile::build(
8836 vec![project.path().to_path_buf()],
8837 Vec::new(),
8838 Vec::new(),
8839 Vec::new(),
8840 Vec::new(),
8841 Vec::new(),
8842 Vec::new(),
8843 sandbox_temp,
8844 )
8845 .unwrap();
8846 let plan = SpawnPlan::launcher_for_test(profile, launcher);
8847 let spawn_marker = project.path().join("spawn-count");
8848 let stop_marker = project.path().join("stop-command");
8849 let quote =
8850 |path: &Path| format!("'{}'", path.display().to_string().replace('\'', "'\\''"));
8851 let command = format!(
8852 "printf 'spawn\\n' >> {}; while [ ! -e {} ]; do sleep 0.05; done",
8853 quote(&spawn_marker),
8854 quote(&stop_marker)
8855 );
8856
8857 let original = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
8858 let task_id = original
8859 .spawn(
8860 plan,
8861 &command,
8862 "sandbox-rehydrate".to_string(),
8863 project.path().to_path_buf(),
8864 HashMap::new(),
8865 Some(Duration::from_secs(30)),
8866 storage.path().to_path_buf(),
8867 10,
8868 true,
8869 false,
8870 Some(project.path().to_path_buf()),
8871 )
8872 .unwrap();
8873 let started = Instant::now();
8874 while !spawn_marker.exists() {
8875 assert!(
8876 started.elapsed() < Duration::from_secs(20),
8877 "original sandboxed task did not start"
8878 );
8879 std::thread::sleep(Duration::from_millis(10));
8880 }
8881 original.detach();
8882
8883 let restarted = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
8884 restarted
8885 .replay_session(storage.path(), "sandbox-rehydrate")
8886 .unwrap();
8887 let replayed = restarted
8888 .status(
8889 &task_id,
8890 "sandbox-rehydrate",
8891 Some(project.path()),
8892 Some(storage.path()),
8893 4096,
8894 )
8895 .expect("rehydrated sandbox task");
8896 assert_eq!(replayed.info.status, BgTaskStatus::Running);
8897 assert!(replayed.sandbox_native);
8898
8899 std::thread::sleep(Duration::from_millis(650));
8900 assert_eq!(
8901 fs::read_to_string(&spawn_marker).unwrap().lines().count(),
8902 1,
8903 "registry replay must observe the persisted process without spawning its command"
8904 );
8905
8906 fs::write(&stop_marker, "stop").unwrap();
8907 let terminal = wait_for_terminal_snapshot(
8908 &restarted,
8909 &task_id,
8910 "sandbox-rehydrate",
8911 project.path(),
8912 storage.path(),
8913 );
8914 assert_eq!(terminal.info.status, BgTaskStatus::Completed);
8915 assert_eq!(
8916 fs::read_to_string(&spawn_marker).unwrap().lines().count(),
8917 1
8918 );
8919 restarted.detach();
8920 }
8921
8922 #[cfg(windows)]
8923 fn wait_for_file(path: &Path) -> String {
8924 let started = Instant::now();
8931 loop {
8932 if let Ok(content) = fs::read_to_string(path) {
8933 if !content.trim().is_empty() {
8934 return content;
8935 }
8936 }
8937 assert!(
8938 started.elapsed() < Duration::from_secs(30),
8939 "timed out waiting for non-empty {}",
8940 path.display()
8941 );
8942 std::thread::sleep(Duration::from_millis(100));
8943 }
8944 }
8945
8946 #[cfg(windows)]
8947 fn spawn_windows_registry_command(
8948 command: &str,
8949 ) -> (BgTaskRegistry, tempfile::TempDir, String) {
8950 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
8951 let dir = tempfile::tempdir().unwrap();
8952 let task_id = registry
8953 .spawn(
8954 SpawnPlan::Unsandboxed,
8955 command,
8956 "session".to_string(),
8957 dir.path().to_path_buf(),
8958 HashMap::new(),
8959 Some(Duration::from_secs(30)),
8960 dir.path().to_path_buf(),
8961 10,
8962 false,
8963 false,
8964 Some(dir.path().to_path_buf()),
8965 )
8966 .unwrap();
8967 (registry, dir, task_id)
8968 }
8969
8970 #[cfg(windows)]
8971 #[test]
8972 fn windows_spawn_writes_exit_marker_for_zero_exit() {
8973 let (registry, _dir, task_id) = spawn_windows_registry_command("cmd /c exit 0");
8974 let exit_path = registry.task_exit_path(&task_id, "session").unwrap();
8975
8976 let content = wait_for_file(&exit_path);
8977
8978 assert_eq!(content.trim(), "0");
8979 }
8980
8981 #[cfg(windows)]
8982 #[test]
8983 fn windows_spawn_writes_exit_marker_for_nonzero_exit() {
8984 let (registry, _dir, task_id) = spawn_windows_registry_command("cmd /c exit 42");
8985 let exit_path = registry.task_exit_path(&task_id, "session").unwrap();
8986
8987 let content = wait_for_file(&exit_path);
8988
8989 assert_eq!(content.trim(), "42");
8990 }
8991
8992 #[cfg(windows)]
8993 #[test]
8994 fn windows_spawn_captures_stdout_to_disk() {
8995 let (registry, _dir, task_id) = spawn_windows_registry_command("cmd /c echo hello");
8996 let task = registry.task_for_session(&task_id, "session").unwrap();
8997 let stdout_path = task.paths.stdout.clone();
8998 let exit_path = task.paths.exit.clone();
8999
9000 let _ = wait_for_file(&exit_path);
9001 let stdout = fs::read_to_string(stdout_path).expect("read stdout");
9002
9003 assert!(stdout.contains("hello"), "stdout was {stdout:?}");
9004 }
9005
9006 #[cfg(windows)]
9007 #[test]
9008 fn windows_spawn_uses_pwsh_when_available() {
9009 let candidates = crate::windows_shell::shell_candidates_with(
9013 |binary| match binary {
9014 "pwsh.exe" => Some(std::path::PathBuf::from(r"C:\pwsh\pwsh.exe")),
9015 "powershell.exe" => Some(std::path::PathBuf::from(r"C:\ps\powershell.exe")),
9016 _ => None,
9017 },
9018 || None,
9019 );
9020 let shell = candidates.first().expect("at least one candidate").clone();
9021 assert_eq!(shell, crate::windows_shell::WindowsShell::Pwsh);
9022 assert_eq!(shell.binary().as_ref(), "pwsh.exe");
9023 }
9024
9025 #[cfg(windows)]
9028 #[test]
9029 fn windows_shell_cmd_wrapper_writes_marker_via_temp_rename() {
9030 let exit_path = Path::new(r"C:\Temp\bash-test.exit");
9031 let script =
9032 crate::windows_shell::WindowsShell::Cmd.wrapper_script("cmd /c exit 42", exit_path);
9033
9034 assert!(
9035 script.contains("set CODE=%ERRORLEVEL%"),
9036 "wrapper must capture the child exit code: {script}"
9037 );
9038 assert!(
9039 script.contains("exit /B %CODE%"),
9040 "wrapper must propagate the child exit code: {script}"
9041 );
9042 assert!(
9047 script.contains("bash-test.exit"),
9048 "wrapper must target the exit marker path: {script}"
9049 );
9050 assert!(
9051 script.contains("move /Y"),
9052 "wrapper must write the marker atomically via temp-file + rename: {script}"
9053 );
9054 }
9055
9056 #[cfg(windows)]
9062 #[test]
9063 fn windows_shell_cmd_bg_command_uses_minimal_cmd_flags() {
9064 use crate::windows_shell::WindowsShell;
9065 let cmd = WindowsShell::Cmd.bg_command("echo wrapped");
9066 let args: Vec<&std::ffi::OsStr> = cmd.get_args().collect();
9067 let args_strs: Vec<&str> = args.iter().filter_map(|a| a.to_str()).collect();
9068 assert_eq!(
9069 args_strs,
9070 vec!["/D", "/S", "/C", "echo wrapped"],
9071 "Cmd::bg_command must prepend /D /S /C"
9072 );
9073 }
9074
9075 #[cfg(windows)]
9078 #[test]
9079 fn windows_shell_pwsh_bg_command_uses_standard_args() {
9080 use crate::windows_shell::WindowsShell;
9081 let cmd = WindowsShell::Pwsh.bg_command("Get-Date");
9082 let args: Vec<&std::ffi::OsStr> = cmd.get_args().collect();
9083 let args_strs: Vec<&str> = args.iter().filter_map(|a| a.to_str()).collect();
9084 assert!(
9085 args_strs.contains(&"-Command"),
9086 "Pwsh::bg_command must use -Command: {args_strs:?}"
9087 );
9088 assert!(
9089 args_strs.contains(&"Get-Date"),
9090 "Pwsh::bg_command must include the user command body"
9091 );
9092 }
9093
9094 fn registry_with_db_and_frames(
9095 storage: &Path,
9096 ) -> (
9097 BgTaskRegistry,
9098 Arc<Mutex<TrackedConnection>>,
9099 Arc<Mutex<Vec<PushFrame>>>,
9100 ) {
9101 let frames = Arc::new(Mutex::new(Vec::new()));
9102 let captured = Arc::clone(&frames);
9103 let sender: crate::context::ProgressSender = Arc::new(Box::new(move |frame| {
9104 captured.lock().unwrap().push(frame);
9105 })
9106 as Box<dyn Fn(PushFrame) + Send + Sync>);
9107 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(Some(sender))));
9108 registry.set_harness(Harness::Opencode);
9109 let conn = crate::db::open(&storage.join("aft.db")).expect("open test DB");
9110 let shared = Arc::new(Mutex::new(conn));
9111 registry.set_db_pool(shared.clone());
9112 (registry, shared, frames)
9113 }
9114
9115 fn pattern_match_frames(frames: &Mutex<Vec<PushFrame>>) -> Vec<BashPatternMatchFrame> {
9116 frames
9117 .lock()
9118 .unwrap()
9119 .iter()
9120 .filter_map(|frame| match frame {
9121 PushFrame::BashPatternMatch(frame) => Some(frame.clone()),
9122 _ => None,
9123 })
9124 .collect()
9125 }
9126
9127 fn install_delivered_terminal_with_pending_watch(
9128 registry: &BgTaskRegistry,
9129 db: &Arc<Mutex<TrackedConnection>>,
9130 storage: &Path,
9131 task_id: &str,
9132 ) -> TaskPaths {
9133 let paths = task_paths(storage, "session", task_id).unwrap();
9134 let mut metadata = PersistedTask::starting(
9135 task_id.to_string(),
9136 "session".to_string(),
9137 "false".to_string(),
9138 storage.to_path_buf(),
9139 Some(storage.to_path_buf()),
9140 None,
9141 true,
9142 true,
9143 );
9144 metadata.mark_terminal(BgTaskStatus::Failed, Some(1), None);
9145 metadata.completion_delivered = true;
9146 write_task(&paths.json, &metadata).unwrap();
9147 {
9148 let conn = db.lock().unwrap();
9149 crate::db::bash_tasks::upsert_bash_task(
9150 &conn,
9151 &metadata.to_bash_task_row("opencode", &paths).unwrap(),
9152 )
9153 .unwrap();
9154 crate::db::bash_watches::upsert_bash_pattern_watch(
9155 &conn,
9156 &BashPatternWatchRow {
9157 harness: "opencode".into(),
9158 session_id: "session".into(),
9159 task_id: task_id.into(),
9160 watch_id: "watch-00000001".into(),
9161 pattern_kind: "substring".into(),
9162 pattern: "(fail)".into(),
9163 once: true,
9164 created_at: 1,
9165 stdout_offset: 756_243,
9166 stderr_offset: 0,
9167 pty_offset: 0,
9168 scanning: false,
9169 pending_match: true,
9170 match_text: Some("(fail)".into()),
9171 match_offset: Some(756_237),
9172 match_context: Some("release output ... (fail)".into()),
9173 },
9174 )
9175 .unwrap();
9176 }
9177 registry
9178 .insert_rehydrated_task(metadata, paths.clone(), true)
9179 .unwrap();
9180 paths
9181 }
9182
9183 #[test]
9188 fn gc_skips_a_task_directory_still_being_created_but_quarantines_an_abandoned_one() {
9189 let dir = tempfile::tempdir().unwrap();
9190 let storage = dir.path();
9191 let registry = BgTaskRegistry::default();
9192 let session_dir = session_tasks_dir(storage, "session");
9193 let young = session_dir.join("bash-0000000000000301");
9194 let abandoned = session_dir.join("bash-0000000000000302");
9195 fs::create_dir_all(&young).unwrap();
9196 fs::create_dir_all(&abandoned).unwrap();
9197 let old = SystemTime::now() - Duration::from_secs(6 * 60);
9198 filetime::set_file_mtime(&abandoned, filetime::FileTime::from_system_time(old)).unwrap();
9199
9200 registry.maybe_gc_persisted(storage).unwrap();
9201
9202 assert!(
9203 young.is_dir(),
9204 "a task directory younger than the grace was quarantined mid-creation"
9205 );
9206 assert!(
9207 !abandoned.is_dir(),
9208 "an empty task directory older than the grace must still be quarantined"
9209 );
9210 let quarantined = fs::read_dir(storage.join("bash-tasks-quarantine"))
9211 .map(|entries| entries.flatten().count())
9212 .unwrap_or(0);
9213 assert_eq!(
9214 quarantined, 1,
9215 "exactly the abandoned layout is quarantined"
9216 );
9217 }
9218
9219 #[test]
9220 fn pending_pattern_match_is_returned_by_drain_contract_until_ack() {
9221 let storage = tempfile::tempdir().unwrap();
9222 let (registry, db, _frames) = registry_with_db_and_frames(storage.path());
9223 let task_id = "bash-0000000000000197";
9224 install_delivered_terminal_with_pending_watch(®istry, &db, storage.path(), task_id);
9225
9226 let first = registry.pending_pattern_matches_for_session("session");
9227 let second = registry.pending_pattern_matches_for_session("session");
9228 assert_eq!(first.len(), 1);
9229 assert_eq!(second.len(), 1);
9230 assert_eq!(first[0].task_id, task_id);
9231 assert_eq!(first[0].watch_id, "watch-00000001");
9232 assert_eq!(registry.unacked_wake_count_for_session(Some("session")), 1);
9233 let stuck =
9234 registry.stuck_pending_watches_for_session("session", Duration::from_secs(10 * 60));
9235 assert_eq!(stuck.len(), 1);
9236 assert_eq!(stuck[0].0, task_id);
9237 assert_eq!(stuck[0].1, "watch-00000001");
9238 assert!(stuck[0].2 >= Duration::from_secs(10 * 60).as_millis() as u64);
9239
9240 assert_eq!(
9241 registry.ack_completions_for_session(Some("session"), &[task_id.to_string()],),
9242 vec![task_id.to_string()]
9243 );
9244 assert!(registry
9245 .pending_pattern_matches_for_session("session")
9246 .is_empty());
9247 assert_eq!(registry.unacked_wake_count_for_session(Some("session")), 0);
9248 }
9249
9250 #[test]
9251 fn terminalized_watch_is_one_shot_across_registry_restart_after_ack() {
9252 let storage = tempfile::tempdir().unwrap();
9253 let (registry, db, _frames) = registry_with_db_and_frames(storage.path());
9254 let task_id = "bash-0000000000000196";
9255 install_delivered_terminal_with_pending_watch(®istry, &db, storage.path(), task_id);
9256 {
9257 let conn = db.lock().unwrap();
9258 let mut row = crate::db::bash_watches::get_bash_pattern_watch(
9259 &conn,
9260 "opencode",
9261 "session",
9262 task_id,
9263 "watch-00000001",
9264 )
9265 .unwrap()
9266 .unwrap();
9267 row.scanning = false;
9268 row.pending_match = true;
9269 row.match_text = Some(WATCH_TARGET_ERASED_TEXT.to_string());
9270 row.match_offset = Some(0);
9271 row.match_context = Some(WATCH_TARGET_ERASED_CONTEXT.to_string());
9272 crate::db::bash_watches::upsert_bash_pattern_watch(&conn, &row).unwrap();
9273 }
9274
9275 let terminalized = registry.pending_pattern_matches_for_session("session");
9276 assert_eq!(terminalized.len(), 1);
9277 assert_eq!(terminalized[0].task_id, task_id);
9278 assert_eq!(terminalized[0].match_text, WATCH_TARGET_ERASED_TEXT);
9279 assert_eq!(
9280 registry.ack_completions_for_session(Some("session"), &[task_id.to_string()]),
9281 vec![task_id.to_string()]
9282 );
9283 drop(registry);
9284
9285 let restarted = BgTaskRegistry::default();
9286 restarted.set_harness(Harness::Opencode);
9287 restarted.set_db_pool(db);
9288 assert!(restarted
9289 .pending_pattern_matches_for_session("session")
9290 .is_empty());
9291 }
9292
9293 #[test]
9294 fn standalone_same_project_multi_session_replay_preserves_earlier_session_delivery() {
9295 let storage = tempfile::tempdir().unwrap();
9296 let project = tempfile::tempdir().unwrap();
9297 let (registry, _db, frames) = registry_with_db_and_frames(storage.path());
9298
9299 registry
9301 .replay_session_for_project(storage.path(), "session-a", project.path())
9302 .unwrap();
9303
9304 let task_watch = registry
9305 .spawn(
9306 SpawnPlan::Unsandboxed,
9307 LONG_RUNNING_COMMAND,
9308 "session-a".to_string(),
9309 project.path().to_path_buf(),
9310 HashMap::new(),
9311 Some(Duration::from_secs(30)),
9312 storage.path().to_path_buf(),
9313 10,
9314 true,
9315 false,
9316 Some(project.path().to_path_buf()),
9317 )
9318 .unwrap();
9319 registry
9320 .register_watch(
9321 task_watch.clone(),
9322 WatchPattern::Substring("READY".into()),
9323 true,
9324 )
9325 .unwrap();
9326
9327 let task_comp = registry
9328 .spawn(
9329 SpawnPlan::Unsandboxed,
9330 LONG_RUNNING_COMMAND,
9331 "session-a".to_string(),
9332 project.path().to_path_buf(),
9333 HashMap::new(),
9334 Some(Duration::from_secs(30)),
9335 storage.path().to_path_buf(),
9336 10,
9337 true,
9338 false,
9339 Some(project.path().to_path_buf()),
9340 )
9341 .unwrap();
9342
9343 registry
9345 .replay_session_for_project(storage.path(), "session-b", project.path())
9346 .unwrap();
9347
9348 let watch_handle = registry.task_for_session(&task_watch, "session-a").unwrap();
9350 std::fs::OpenOptions::new()
9351 .append(true)
9352 .open(&watch_handle.paths.stdout)
9353 .unwrap()
9354 .write_all(b"READY\n")
9355 .unwrap();
9356 registry.scan_task_watch_output(&watch_handle);
9357
9358 let comp_handle = registry.task_for_session(&task_comp, "session-a").unwrap();
9360 fs::write(&comp_handle.paths.exit, "0\n").unwrap();
9361 comp_handle.mark_terminal_now();
9362 comp_handle.state.lock().unwrap().metadata.mark_terminal(
9363 BgTaskStatus::Completed,
9364 Some(0),
9365 None,
9366 );
9367 registry
9368 .post_terminal_transition(&comp_handle, true)
9369 .unwrap();
9370
9371 let deadline = Instant::now() + Duration::from_secs(5);
9375 loop {
9376 let emitted = frames.lock().unwrap().iter().any(
9377 |frame| matches!(frame, PushFrame::BashCompleted(f) if f.task_id == task_comp),
9378 );
9379 if emitted || Instant::now() >= deadline {
9380 break;
9381 }
9382 std::thread::sleep(Duration::from_millis(20));
9383 }
9384
9385 let captured = frames.lock().unwrap();
9386 let pattern_matches = captured
9387 .iter()
9388 .filter_map(|frame| match frame {
9389 PushFrame::BashPatternMatch(frame) => Some(frame),
9390 _ => None,
9391 })
9392 .collect::<Vec<_>>();
9393 assert!(
9394 pattern_matches.iter().any(|f| f.task_id == task_watch
9395 && f.session_id == "session-a"
9396 && f.match_text == "READY"),
9397 "session A pattern match must be emitted after session B replay: {pattern_matches:?}"
9398 );
9399
9400 let completions = captured
9401 .iter()
9402 .filter_map(|frame| match frame {
9403 PushFrame::BashCompleted(frame) => Some(frame),
9404 _ => None,
9405 })
9406 .collect::<Vec<_>>();
9407 assert!(
9408 completions
9409 .iter()
9410 .any(|f| f.task_id == task_comp && f.session_id == "session-a"),
9411 "session A completion must be emitted after session B replay: {completions:?}"
9412 );
9413 }
9414
9415 #[cfg(unix)]
9416 #[test]
9417 fn gc_refuses_to_delete_or_quarantine_a_recorded_live_process() {
9418 let dir = tempfile::tempdir().unwrap();
9419 let storage = dir.path();
9420 let (registry, db, _frames) = registry_with_db_and_frames(storage);
9421 let task_id = "bash-0000000000000198";
9422 let paths = task_paths(storage, "session", task_id).unwrap();
9423 let mut running = PersistedTask::starting(
9424 task_id.to_string(),
9425 "session".to_string(),
9426 "live-process-canary".to_string(),
9427 storage.to_path_buf(),
9428 Some(storage.to_path_buf()),
9429 None,
9430 true,
9431 false,
9432 );
9433 running.mark_running(std::process::id(), std::process::id() as i32);
9437 write_task(&paths.json, &running).unwrap();
9438 fs::write(&paths.stdout, b"").unwrap();
9439 fs::write(&paths.stderr, b"").unwrap();
9440 {
9441 let conn = db.lock().unwrap();
9442 crate::db::bash_tasks::upsert_bash_task(
9443 &conn,
9444 &running.to_bash_task_row("opencode", &paths).unwrap(),
9445 )
9446 .unwrap();
9447 }
9448 let running_json = fs::read(&paths.json).unwrap();
9449 let mut terminal: PersistedTask = serde_json::from_slice(&running_json).unwrap();
9450 terminal.mark_terminal(BgTaskStatus::Completed, Some(0), None);
9451 terminal.completion_delivered = true;
9452 write_task_at(
9453 &resolve_task_layout(&paths.session_dir, task_id).unwrap(),
9454 &terminal,
9455 )
9456 .unwrap();
9457 let old = SystemTime::now()
9458 .checked_sub(Duration::from_secs(25 * 60 * 60))
9459 .unwrap();
9460 filetime::set_file_mtime(&paths.json, filetime::FileTime::from_system_time(old)).unwrap();
9461
9462 assert_eq!(registry.maybe_gc_persisted(storage).unwrap(), 0);
9463 assert!(paths.io_dir.exists(), "GC deleted a live task bundle");
9464
9465 fs::write(&paths.json, b"{corrupt").unwrap();
9466 filetime::set_file_mtime(&paths.json, filetime::FileTime::from_system_time(old)).unwrap();
9467 assert_eq!(registry.maybe_gc_persisted(storage).unwrap(), 0);
9468 assert!(
9469 paths.io_dir.exists(),
9470 "GC quarantined a live task with unreadable metadata"
9471 );
9472
9473 fs::write(&paths.json, running_json).unwrap();
9474 }
9475
9476 #[test]
9477 fn pattern_watch_survives_registry_teardown_and_rehydrate() {
9478 let dir = tempfile::tempdir().unwrap();
9479 let storage = dir.path();
9480 let (registry, _db, frames) = registry_with_db_and_frames(storage);
9481 let task_id = registry
9482 .spawn(
9483 SpawnPlan::Unsandboxed,
9484 LONG_RUNNING_COMMAND,
9485 "session".to_string(),
9486 storage.to_path_buf(),
9487 HashMap::new(),
9488 Some(Duration::from_secs(30)),
9489 storage.to_path_buf(),
9490 10,
9491 true,
9492 false,
9493 Some(storage.to_path_buf()),
9494 )
9495 .unwrap();
9496 registry
9497 .register_watch(
9498 task_id.clone(),
9499 WatchPattern::Substring("READY".into()),
9500 true,
9501 )
9502 .unwrap();
9503 let task = registry.task_for_session(&task_id, "session").unwrap();
9504 registry.clear_task_watch_state(&task_id);
9506 assert_eq!(registry.active_watch_count(&task_id), 0);
9507
9508 std::fs::OpenOptions::new()
9509 .append(true)
9510 .open(&task.paths.stdout)
9511 .unwrap()
9512 .write_all(b"READY\n")
9513 .unwrap();
9514 frames.lock().unwrap().clear();
9515
9516 let (replayed, _db2, replay_frames) = registry_with_db_and_frames(storage);
9517 registry
9519 .inner
9520 .shutdown
9521 .store(true, std::sync::atomic::Ordering::SeqCst);
9522 replayed
9523 .replay_session_inner(storage, "session", None)
9524 .unwrap();
9525
9526 let matches = pattern_match_frames(&replay_frames);
9527 assert!(
9528 matches.iter().any(|frame| {
9529 frame.task_id == task_id
9530 && frame.reason == "pattern_match"
9531 && frame.match_text == "READY"
9532 }),
9533 "rehydrate should deliver gap match: {matches:?}"
9534 );
9535 }
9536
9537 #[test]
9538 fn pattern_watch_gap_match_between_teardown_and_rehydrate_delivers_once() {
9539 let dir = tempfile::tempdir().unwrap();
9540 let storage = dir.path();
9541 let (registry, _db, frames) = registry_with_db_and_frames(storage);
9542 let task_id = registry
9543 .spawn(
9544 SpawnPlan::Unsandboxed,
9545 LONG_RUNNING_COMMAND,
9546 "session".to_string(),
9547 storage.to_path_buf(),
9548 HashMap::new(),
9549 Some(Duration::from_secs(30)),
9550 storage.to_path_buf(),
9551 10,
9552 true,
9553 false,
9554 Some(storage.to_path_buf()),
9555 )
9556 .unwrap();
9557 registry
9558 .register_watch(
9559 task_id.clone(),
9560 WatchPattern::Substring("GAP-HIT".into()),
9561 true,
9562 )
9563 .unwrap();
9564 let task = registry.task_for_session(&task_id, "session").unwrap();
9565 let cursor_before = registry.watch_stream_cursors(&task_id).0;
9566 registry.clear_task_watch_state(&task_id);
9567
9568 std::fs::OpenOptions::new()
9570 .append(true)
9571 .open(&task.paths.stdout)
9572 .unwrap()
9573 .write_all(b"prefix GAP-HIT suffix\n")
9574 .unwrap();
9575 frames.lock().unwrap().clear();
9576
9577 let (replayed, _db2, replay_frames) = registry_with_db_and_frames(storage);
9578 registry
9579 .inner
9580 .shutdown
9581 .store(true, std::sync::atomic::Ordering::SeqCst);
9582 replayed
9583 .replay_session_inner(storage, "session", None)
9584 .unwrap();
9585
9586 let matches: Vec<_> = pattern_match_frames(&replay_frames)
9587 .into_iter()
9588 .filter(|frame| frame.task_id == task_id && frame.match_text.contains("GAP-HIT"))
9589 .collect();
9590 assert_eq!(
9591 matches.len(),
9592 1,
9593 "gap match must deliver exactly once: {matches:?}"
9594 );
9595 assert!(
9596 matches[0].match_offset >= cursor_before,
9597 "match offset should be at/after the persisted cursor ({cursor_before}), got {}",
9598 matches[0].match_offset
9599 );
9600 }
9601
9602 #[test]
9603 fn pattern_watch_acked_match_does_not_redeliver_after_restart() {
9604 let dir = tempfile::tempdir().unwrap();
9605 let storage = dir.path();
9606 let (registry, db, frames) = registry_with_db_and_frames(storage);
9607 let task_id = registry
9608 .spawn(
9609 SpawnPlan::Unsandboxed,
9610 LONG_RUNNING_COMMAND,
9611 "session".to_string(),
9612 storage.to_path_buf(),
9613 HashMap::new(),
9614 Some(Duration::from_secs(30)),
9615 storage.to_path_buf(),
9616 10,
9617 true,
9618 false,
9619 Some(storage.to_path_buf()),
9620 )
9621 .unwrap();
9622 registry
9623 .register_watch(
9624 task_id.clone(),
9625 WatchPattern::Substring("READY".into()),
9626 true,
9627 )
9628 .unwrap();
9629 let task = registry.task_for_session(&task_id, "session").unwrap();
9630 std::fs::OpenOptions::new()
9631 .append(true)
9632 .open(&task.paths.stdout)
9633 .unwrap()
9634 .write_all(b"READY\n")
9635 .unwrap();
9636 registry.scan_task_watch_output(&task);
9637 let delivered = pattern_match_frames(&frames);
9638 assert!(
9639 delivered
9640 .iter()
9641 .any(|frame| frame.task_id == task_id && frame.match_text == "READY"),
9642 "live path should deliver match: {delivered:?}"
9643 );
9644 assert!(registry
9646 .ack_completions_for_session(Some("session"), std::slice::from_ref(&task_id))
9647 .contains(&task_id));
9648 {
9649 let conn = db.lock().unwrap();
9650 let rows = crate::db::bash_watches::list_bash_pattern_watches_for_task(
9651 &conn, "opencode", "session", &task_id,
9652 )
9653 .unwrap();
9654 assert!(
9655 rows.is_empty(),
9656 "acked once-watch rows must be deleted: {rows:?}"
9657 );
9658 }
9659
9660 frames.lock().unwrap().clear();
9661 let (replayed, _db2, replay_frames) = registry_with_db_and_frames(storage);
9662 registry
9663 .inner
9664 .shutdown
9665 .store(true, std::sync::atomic::Ordering::SeqCst);
9666 replayed
9667 .replay_session_inner(storage, "session", None)
9668 .unwrap();
9669 let matches = pattern_match_frames(&replay_frames)
9670 .into_iter()
9671 .filter(|frame| frame.task_id == task_id)
9672 .collect::<Vec<_>>();
9673 assert!(
9674 matches.is_empty(),
9675 "acked match must not re-deliver after restart: {matches:?}"
9676 );
9677 }
9678
9679 #[test]
9680 fn pending_watch_match_redelivers_after_terminal_cleanup_removes_task_bundle() {
9681 let dir = tempfile::tempdir().unwrap();
9682 let storage = dir.path();
9683 let (registry, db, frames) = registry_with_db_and_frames(storage);
9684 let task_id = "bash-aaaaaaaaaaaaaaa1";
9685 let paths = install_delivered_terminal_with_pending_watch(®istry, &db, storage, task_id);
9686 frames.lock().unwrap().clear();
9687
9688 registry.cleanup_finished(Duration::ZERO);
9689 assert!(registry.task(task_id).is_none());
9690 assert!(
9691 !paths.json.exists(),
9692 "cleanup must remove the task metadata bundle"
9693 );
9694
9695 let _ = registry.drain_completions_for_session(Some("session"));
9696 let matches = pattern_match_frames(&frames);
9697 assert!(
9698 matches.iter().any(|frame| {
9699 frame.task_id == task_id
9700 && frame.watch_id == "watch-00000001"
9701 && frame.match_text == "(fail)"
9702 }),
9703 "durable pending match must redeliver without an in-memory task: {matches:?}"
9704 );
9705 assert!(registry
9706 .ack_completions_for_session(Some("session"), &[task_id.to_string()])
9707 .contains(&task_id.to_string()));
9708 let rows = crate::db::bash_watches::list_bash_pattern_watches_for_task(
9709 &db.lock().unwrap(),
9710 "opencode",
9711 "session",
9712 task_id,
9713 )
9714 .unwrap();
9715 assert!(rows.is_empty(), "ack must end durable redelivery");
9716 }
9717
9718 #[test]
9719 fn status_uses_intact_terminal_db_row_after_task_bundle_cleanup() {
9720 let dir = tempfile::tempdir().unwrap();
9721 let storage = dir.path();
9722 let (registry, db, _frames) = registry_with_db_and_frames(storage);
9723 let task_id = "bash-aaaaaaaaaaaaaaa2";
9724 let paths = install_delivered_terminal_with_pending_watch(®istry, &db, storage, task_id);
9725
9726 registry.cleanup_finished(Duration::ZERO);
9727 assert!(registry.task(task_id).is_none());
9728 assert!(
9729 !paths.json.exists(),
9730 "cleanup must remove the task metadata bundle"
9731 );
9732 assert!(
9733 crate::db::bash_tasks::get_bash_task(
9734 &db.lock().unwrap(),
9735 "opencode",
9736 "session",
9737 task_id
9738 )
9739 .unwrap()
9740 .is_some(),
9741 "cleanup must retain the terminal database row"
9742 );
9743
9744 let snapshot = registry
9745 .status(
9746 task_id,
9747 "session",
9748 Some(storage),
9749 Some(storage),
9750 RUNNING_OUTPUT_PREVIEW_BYTES,
9751 )
9752 .expect("intact terminal row must remain visible after artifact cleanup");
9753 assert_eq!(snapshot.info.status, BgTaskStatus::Failed);
9754 assert_eq!(snapshot.exit_code, Some(1));
9755 assert!(snapshot.info.duration_ms.is_some());
9756 }
9757
9758 #[test]
9759 fn pattern_watch_rows_are_deleted_when_task_is_gc_deleted() {
9760 let dir = tempfile::tempdir().unwrap();
9761 let storage = dir.path();
9762 let (registry, db, _frames) = registry_with_db_and_frames(storage);
9763 let task_id = "bash-aaaaaaaaaaaaaaaa";
9764 let paths = task_paths(storage, "session", task_id).unwrap();
9765 let mut metadata = PersistedTask::starting(
9766 task_id.to_string(),
9767 "session".to_string(),
9768 "true".to_string(),
9769 storage.to_path_buf(),
9770 Some(storage.to_path_buf()),
9771 None,
9772 true,
9773 true,
9774 );
9775 metadata.mark_terminal(BgTaskStatus::Completed, Some(0), None);
9776 metadata.completion_delivered = true;
9777 write_task(&paths.json, &metadata).unwrap();
9778 {
9779 let conn = db.lock().unwrap();
9780 crate::db::bash_tasks::upsert_bash_task(
9781 &conn,
9782 &metadata.to_bash_task_row("opencode", &paths).unwrap(),
9783 )
9784 .unwrap();
9785 crate::db::bash_watches::upsert_bash_pattern_watch(
9786 &conn,
9787 &BashPatternWatchRow {
9788 harness: "opencode".into(),
9789 session_id: "session".into(),
9790 task_id: task_id.into(),
9791 watch_id: "watch-00000001".into(),
9792 pattern_kind: "substring".into(),
9793 pattern: "x".into(),
9794 once: true,
9795 created_at: 1,
9796 stdout_offset: 0,
9797 stderr_offset: 0,
9798 pty_offset: 0,
9799 scanning: true,
9800 pending_match: false,
9801 match_text: None,
9802 match_offset: None,
9803 match_context: None,
9804 },
9805 )
9806 .unwrap();
9807 }
9808 let old = SystemTime::now()
9809 .checked_sub(Duration::from_secs(25 * 60 * 60))
9810 .unwrap();
9811 filetime::set_file_mtime(&paths.json, filetime::FileTime::from_system_time(old)).unwrap();
9812
9813 let deleted = registry.maybe_gc_persisted(storage).unwrap();
9814 assert!(
9815 deleted >= 1,
9816 "expected GC to delete the terminal task bundle"
9817 );
9818 let conn = db.lock().unwrap();
9819 let watches = crate::db::bash_watches::list_bash_pattern_watches_for_task(
9820 &conn, "opencode", "session", task_id,
9821 )
9822 .unwrap();
9823 assert!(
9824 watches.is_empty(),
9825 "task-row GC must cascade to every persisted watch"
9826 );
9827 }
9828}