1use std::collections::{BTreeMap, HashMap, HashSet, VecDeque};
2#[cfg(not(windows))]
3use std::ffi::OsString;
4use std::fs;
5use std::io::Write;
6use std::path::{Path, PathBuf};
7#[cfg(windows)]
8use std::process::Command;
9use std::process::{Child, Stdio};
10use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
11#[cfg(unix)]
12use std::sync::OnceLock;
13use std::sync::{Arc, Mutex, RwLock};
14use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
15
16use crate::db::TrackedConnection;
17use serde::Serialize;
18
19use crate::bash_permissions::PermissionAsk;
20use crate::compress::caps::DropClass;
21#[cfg(unix)]
22use crate::compress::single_top_level_pipeline;
23use crate::compress::CompressionResult;
24use crate::context::SharedProgressSender;
25use crate::db::compression_events::CompressionAggregateCache;
26use crate::harness::Harness;
27use crate::list_envelope::ListEnvelope;
28use crate::protocol::{BashCompletedFrame, BashLongRunningFrame, BashPatternMatchFrame, PushFrame};
29use crate::sandbox_spawn::SpawnPlan;
30
31#[cfg(windows)]
32use std::os::windows::process::CommandExt;
33
34use super::buffer::{combine_streams, BgBuffer, DiskTruncation, StreamKind, TokenCountInput};
35use super::output::{
36 cap_completion_output, cap_completion_output_with_marker, cap_final_output,
37 cap_final_output_with_marker, completion_preview_threshold, json_output_pointer, quote_path,
38 retained_json_output_pointer, COMPRESS_INPUT_CAP_BYTES, COMPRESS_INPUT_HEAD_BYTES,
39 COMPRESS_INPUT_TAIL_BYTES, FINAL_OUTPUT_CAP_BYTES, RAW_PASSTHROUGH_CAP_BYTES,
40 RAW_PASSTHROUGH_HEAD_BYTES, RAW_PASSTHROUGH_TAIL_BYTES, RUNNING_OUTPUT_PREVIEW_BYTES,
41 STRUCTURED_OUTPUT_CAP_BYTES,
42};
43use super::persistence::{
44 allocate_task_layout, delete_resolved_task, delete_task_bundle, discover_task_ids,
45 open_task_artifact, quarantine_invalid_entry, quarantine_task_layout, read_exit_marker,
46 read_task_at, resolve_task_layout, session_tasks_dir, uninitialized_layout_is_recent,
47 unix_millis, update_task_at, validate_task_id, write_kill_marker_if_absent, write_task_at,
48 BgMode, ExitMarker, PersistedTask, TaskArtifact, TaskIoHandles, TaskPaths,
49};
50#[cfg(unix)]
51use super::process::terminate_pgid;
52#[cfg(windows)]
53use super::process::terminate_pid;
54use super::process::{
55 is_process_alive, is_recorded_process_alive, live_process_group_members, LiveDescendant,
56};
57use super::pty_process::spawn_pty_for_command;
58use super::pty_runtime::PtyRuntime;
59use super::watches::{
60 PatternMatch, WatchPattern, WatchRegistry, WATCH_TARGET_ERASED_CONTEXT,
61 WATCH_TARGET_ERASED_TEXT, WATCH_TASK_EXIT_TEXT,
62};
63use super::{BgTaskInfo, BgTaskStatus};
64use crate::db::bash_tasks::BashTaskRow;
65use crate::db::bash_watches::BashPatternWatchRow;
66const DEFAULT_BG_TIMEOUT: Duration = Duration::from_secs(30 * 60);
69const PERSISTED_GC_GRACE: Duration = Duration::from_secs(24 * 60 * 60);
70const QUARANTINE_GC_GRACE: Duration = Duration::from_secs(30 * 24 * 60 * 60);
71
72const TOKENIZE_CAP_BYTES_PER_STREAM: usize = 128 * 1024;
73pub const ROOT_RECLAIMED_REASON: &str = "root_reclaimed";
74#[cfg(target_os = "linux")]
75pub(crate) const LINUX_SCOPE_ENV: &str = "AFT_INTERNAL_LINUX_SCOPE";
76
77#[derive(Debug, Clone, Serialize)]
78pub struct BgCompletion {
79 pub task_id: String,
80 #[serde(skip_serializing)]
83 pub session_id: String,
84 pub status: BgTaskStatus,
85 pub exit_code: Option<i32>,
86 pub command: String,
87 #[serde(default, skip_serializing_if = "String::is_empty")]
93 pub output_preview: String,
94 #[serde(default, skip_serializing_if = "Option::is_none")]
95 pub bash_output_list_envelope: Option<ListEnvelope>,
96 #[serde(default, skip_serializing_if = "is_false")]
101 pub output_truncated: bool,
102 #[serde(default, skip_serializing_if = "Option::is_none")]
105 pub original_tokens: Option<u32>,
106 #[serde(default, skip_serializing_if = "Option::is_none")]
109 pub compressed_tokens: Option<u32>,
110 #[serde(default, skip_serializing_if = "is_false")]
112 pub tokens_skipped: bool,
113 #[serde(default, skip_serializing_if = "Option::is_none")]
114 pub status_reason: Option<String>,
115 pub live_descendants: Option<Vec<LiveDescendant>>,
116 #[serde(default, skip_serializing_if = "is_zero_usize")]
117 pub live_descendants_omitted: usize,
118 #[serde(default, skip_serializing_if = "Option::is_none")]
119 pub live_descendants_summary: Option<String>,
120}
121
122fn is_zero_usize(value: &usize) -> bool {
123 *value == 0
124}
125
126fn is_false(v: &bool) -> bool {
127 !*v
128}
129
130#[derive(Debug, Clone, Serialize)]
131pub struct BgTaskSnapshot {
132 #[serde(flatten)]
133 pub info: BgTaskInfo,
134 pub exit_code: Option<i32>,
135 pub child_pid: Option<u32>,
136 pub workdir: String,
137 pub output_preview: String,
138 #[serde(default, skip_serializing_if = "Option::is_none")]
139 pub bash_output_list_envelope: Option<ListEnvelope>,
140 pub output_truncated: bool,
141 pub output_path: Option<String>,
142 pub stderr_path: Option<String>,
143 #[serde(skip_serializing_if = "Option::is_none")]
144 pub pty_rows: Option<u16>,
145 #[serde(skip_serializing_if = "Option::is_none")]
146 pub pty_cols: Option<u16>,
147 #[serde(skip_serializing_if = "Option::is_none")]
148 pub pty_screen: Option<String>,
149 #[serde(default, skip_serializing_if = "Vec::is_empty")]
150 pub scanner_report: Vec<PermissionAsk>,
151 #[serde(default, skip_serializing_if = "is_false")]
152 pub sandbox_native: bool,
153 #[serde(default, skip_serializing_if = "is_false")]
154 pub sandbox_unavailable: bool,
155 pub live_descendants: Option<Vec<LiveDescendant>>,
156 #[serde(default, skip_serializing_if = "is_zero_usize")]
157 pub live_descendants_omitted: usize,
158 #[serde(default, skip_serializing_if = "Option::is_none")]
159 pub live_descendants_summary: Option<String>,
160 #[serde(default, skip_serializing_if = "is_false")]
161 pub kill_signaled: bool,
162 #[serde(default, skip_serializing_if = "is_zero_usize")]
163 pub kill_reached: usize,
164}
165
166#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
167pub struct BgTaskHealthCounts {
168 pub running: usize,
169 pub pending_completions: usize,
170}
171
172#[derive(Debug, Clone, Copy, PartialEq, Eq)]
173enum TerminalOutputKind {
174 Compressed,
175 Raw,
176 Structured,
177}
178
179#[derive(Debug, Clone, PartialEq, Eq)]
180struct TerminalOutputCache {
181 output_preview: String,
182 output_truncated: bool,
183 compression_input_line_count: Option<usize>,
184 kind: TerminalOutputKind,
185 output_path: Option<String>,
186 stderr_path: Option<String>,
187 artifact_access: ArtifactRecoveryAccess,
188 recovery: Option<RecoveryContext>,
189}
190
191#[derive(Debug, Clone, PartialEq, Eq)]
192struct ArtifactRecoveryAccess {
193 task_id: String,
194 readable: bool,
195}
196
197#[derive(Debug, Clone, PartialEq, Eq)]
198struct RecoveryContext {
199 dropped_by_class: BTreeMap<DropClass, usize>,
200 had_inner_drop: bool,
201 offset_hint_eligible: bool,
202 offset_start_line: Option<usize>,
203 byte_truncated: bool,
204 disk_truncated_prefix_bytes: u64,
205 output_path: Option<String>,
206 stderr_path: Option<String>,
207 include_stderr_path: bool,
208 artifact_access: ArtifactRecoveryAccess,
209}
210
211fn optional_string_bytes(value: Option<&String>) -> u64 {
212 value
213 .map(|value| crate::memory::usize_to_u64(value.len()))
214 .unwrap_or(0)
215}
216
217fn terminal_output_cache_estimated_bytes(cache: &TerminalOutputCache) -> u64 {
218 let recovery_bytes = cache
219 .recovery
220 .as_ref()
221 .map(|recovery| {
222 crate::memory::usize_to_u64(recovery.dropped_by_class.len())
223 .saturating_mul(
224 (std::mem::size_of::<DropClass>() + std::mem::size_of::<usize>()) as u64,
225 )
226 .saturating_add(optional_string_bytes(recovery.output_path.as_ref()))
227 .saturating_add(optional_string_bytes(recovery.stderr_path.as_ref()))
228 .saturating_add(crate::memory::usize_to_u64(
229 recovery.artifact_access.task_id.len(),
230 ))
231 })
232 .unwrap_or(0);
233 (std::mem::size_of::<TerminalOutputCache>() as u64)
234 .saturating_add(crate::memory::usize_to_u64(cache.output_preview.len()))
235 .saturating_add(optional_string_bytes(cache.output_path.as_ref()))
236 .saturating_add(optional_string_bytes(cache.stderr_path.as_ref()))
237 .saturating_add(crate::memory::usize_to_u64(
238 cache.artifact_access.task_id.len(),
239 ))
240 .saturating_add(recovery_bytes)
241}
242
243fn completion_estimated_bytes(completion: &BgCompletion) -> u64 {
244 (std::mem::size_of::<BgCompletion>() as u64)
245 .saturating_add(crate::memory::usize_to_u64(completion.task_id.len()))
246 .saturating_add(crate::memory::usize_to_u64(completion.session_id.len()))
247 .saturating_add(crate::memory::usize_to_u64(completion.command.len()))
248 .saturating_add(crate::memory::usize_to_u64(completion.output_preview.len()))
249}
250
251impl RecoveryContext {
252 fn has_visible_drop(&self) -> bool {
253 self.byte_truncated
254 || self.disk_truncated_prefix_bytes > 0
255 || self.had_inner_drop
256 || !self.dropped_by_class.is_empty()
257 }
258}
259
260#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
268struct SessionRecovery {
269 in_flight: usize,
270 finished: u64,
271}
272
273#[doc(hidden)]
278pub struct SessionRecoveryGuard {
279 registry: BgTaskRegistry,
280 key: Option<String>,
281}
282
283impl Drop for SessionRecoveryGuard {
284 fn drop(&mut self) {
285 let Some(key) = self.key.take() else {
286 return;
287 };
288 let mut recoveries = self
289 .registry
290 .inner
291 .session_recovery
292 .lock()
293 .unwrap_or_else(std::sync::PoisonError::into_inner);
294 let Some(recovery) = recoveries.get_mut(&key) else {
295 return;
296 };
297 recovery.in_flight = recovery.in_flight.saturating_sub(1);
298 recovery.finished = recovery.finished.wrapping_add(1);
299 }
303}
304
305#[derive(Clone)]
306pub struct BgTaskRegistry {
307 pub(crate) inner: Arc<RegistryInner>,
308}
309
310#[derive(Debug, Clone, Copy, PartialEq, Eq)]
312pub enum WatchdogPassCause {
313 Wake,
315 Tick,
317}
318
319pub(crate) struct RegistryInner {
320 pub(crate) tasks: Mutex<HashMap<String, Arc<BgTask>>>,
321 pub(crate) completions: Mutex<VecDeque<BgCompletion>>,
322 pub(crate) progress_sender: SharedProgressSender,
323 watchdog_started: AtomicBool,
324 pub(crate) shutdown: AtomicBool,
325 pub(crate) long_running_reminder_enabled: AtomicBool,
326 pub(crate) long_running_reminder_interval_ms: AtomicU64,
327 persisted_gc_started: AtomicBool,
328 #[cfg(test)]
329 persisted_gc_runs: AtomicU64,
330 persisted_gc_thread: Mutex<Option<String>>,
334 session_recovery: Mutex<HashMap<String, SessionRecovery>>,
339 pub(crate) compressor:
345 Mutex<Option<Box<dyn Fn(&str, String, Option<i32>) -> CompressionResult + Send + Sync>>>,
346 pub(crate) db_pool: RwLock<Option<Arc<Mutex<TrackedConnection>>>>,
347 pub(crate) db_harness: RwLock<Option<String>>,
348 pub(crate) compression_aggregates: Arc<CompressionAggregateCache>,
349 pub(crate) wake_tx: crossbeam_channel::Sender<()>,
350 pub(crate) wake_rx: crossbeam_channel::Receiver<()>,
351 terminal_transition: tokio::sync::Notify,
352 pub(crate) completion_pass_cause: Mutex<HashMap<String, WatchdogPassCause>>,
356 pub(crate) watch_registry: Mutex<WatchRegistry>,
357 live_delivery_sessions: Mutex<HashSet<String>>,
362 wait_detach_sessions: Mutex<HashSet<String>>,
363 active_wait_sessions: Mutex<HashMap<String, usize>>,
364 wait_registered_tasks: Mutex<HashMap<String, HashSet<String>>>,
365}
366
367pub(crate) struct BgTask {
368 pub(crate) task_id: String,
369 pub(crate) session_id: String,
370 pub(crate) paths: TaskPaths,
371 artifact_root: PathBuf,
372 pub(crate) started: Instant,
373 pub(crate) last_reminder_at: Mutex<Option<Instant>>,
374 pub(crate) terminal_at: Mutex<Option<Instant>>,
375 pub(crate) state: Mutex<BgTaskState>,
376}
377
378pub(crate) enum TaskRuntime {
379 Piped(Option<Child>),
380 Pty(Option<PtyRuntime>),
381}
382
383pub(crate) struct BgTaskState {
384 pub(crate) metadata: PersistedTask,
385 pub(crate) runtime: TaskRuntime,
386 pub(crate) io_handles: Option<TaskIoHandles>,
389 pub(crate) detached: bool,
390 pub(crate) child_exit_observed: bool,
401 pub(crate) descendant_sampling_started: bool,
404 pub(crate) buffer: BgBuffer,
405 terminal_output_cache: Option<TerminalOutputCache>,
406 pub(crate) pending_terminal_override: Option<BgTaskStatus>,
408}
409
410fn completion_matches_session(completion: &BgCompletion, session_id: Option<&str>) -> bool {
411 session_id
412 .map(|session_id| completion.session_id == session_id)
413 .unwrap_or(true)
414}
415
416impl BgTaskRegistry {
417 pub fn completion_pass_cause(&self, task_id: &str) -> Option<WatchdogPassCause> {
420 self.inner
421 .completion_pass_cause
422 .lock()
423 .ok()
424 .and_then(|map| map.get(task_id).copied())
425 }
426
427 pub fn new(progress_sender: SharedProgressSender) -> Self {
428 let (wake_tx, wake_rx) = crossbeam_channel::bounded(1);
429 Self {
430 inner: Arc::new(RegistryInner {
431 tasks: Mutex::new(HashMap::new()),
432 completions: Mutex::new(VecDeque::new()),
433 progress_sender,
434 watchdog_started: AtomicBool::new(false),
435 shutdown: AtomicBool::new(false),
436 long_running_reminder_enabled: AtomicBool::new(true),
437 long_running_reminder_interval_ms: AtomicU64::new(600_000),
438 persisted_gc_started: AtomicBool::new(false),
439 #[cfg(test)]
440 persisted_gc_runs: AtomicU64::new(0),
441 persisted_gc_thread: Mutex::new(None),
442 session_recovery: Mutex::new(HashMap::new()),
443 compressor: Mutex::new(None),
444 db_pool: RwLock::new(None),
445 db_harness: RwLock::new(None),
446 compression_aggregates: Arc::new(CompressionAggregateCache::default()),
447 wake_tx,
448 wake_rx,
449 terminal_transition: tokio::sync::Notify::new(),
450 completion_pass_cause: Mutex::new(HashMap::new()),
451 watch_registry: Mutex::new(WatchRegistry::default()),
452 live_delivery_sessions: Mutex::new(HashSet::new()),
453 wait_detach_sessions: Mutex::new(HashSet::new()),
454 active_wait_sessions: Mutex::new(HashMap::new()),
455 wait_registered_tasks: Mutex::new(HashMap::new()),
456 }),
457 }
458 }
459
460 pub fn is_session_owned_artifact_path(&self, session_id: &str, path: &Path) -> bool {
468 let Ok(requested) = fs::canonicalize(path) else {
469 return false;
470 };
471 let Ok(tasks) = self.inner.tasks.lock() else {
472 return false;
473 };
474
475 tasks.values().any(|task| {
476 task.session_id == session_id
477 && [
478 &task.paths.stdout,
479 &task.paths.stderr,
480 &task.paths.exit,
481 &task.paths.pty,
482 ]
483 .into_iter()
484 .filter_map(|known| known.file_name())
485 .any(|name| task.artifact_root.join(name) == requested)
486 })
487 }
488
489 pub fn read_artifact_path(
490 &self,
491 session_id: &str,
492 path: &Path,
493 ) -> Option<Result<Vec<u8>, String>> {
494 let requested = fs::canonicalize(path).ok()?;
495 let tasks = self.inner.tasks.lock().ok()?;
496 let (task, artifact) = tasks.values().find_map(|task| {
497 if task.session_id != session_id {
498 return None;
499 }
500 TaskArtifact::ALL.into_iter().find_map(|artifact| {
501 let expected = task
502 .paths
503 .artifact_path(artifact)
504 .file_name()
505 .map(|name| task.artifact_root.join(name));
506 (expected.as_deref() == Some(requested.as_path()))
507 .then(|| (Arc::clone(task), artifact))
508 })
509 })?;
510 drop(tasks);
511 Some(self.read_artifact(&task.task_id, session_id, artifact))
512 }
513
514 pub fn read_artifact_range(
515 &self,
516 task_id: &str,
517 session_id: &str,
518 artifact: TaskArtifact,
519 offset: u64,
520 ) -> Result<(Vec<u8>, u64), String> {
521 validate_task_id(task_id).map_err(|error| error.to_string())?;
522 let task = self
523 .task_for_session(task_id, session_id)
524 .ok_or_else(|| "task_not_found".to_string())?;
525 let mut file = open_task_artifact(&task.paths, artifact)
526 .map_err(|error| format!("artifact_refused: {error}"))?;
527 let len = file
528 .len()
529 .map_err(|error| format!("artifact_refused: {error}"))?;
530 let start = offset.min(len);
531 let bytes = file
532 .read_range(start, len.saturating_sub(start))
533 .map_err(|error| format!("artifact_refused: {error}"))?;
534 Ok((bytes, len))
535 }
536
537 pub fn read_artifact(
538 &self,
539 task_id: &str,
540 session_id: &str,
541 artifact: TaskArtifact,
542 ) -> Result<Vec<u8>, String> {
543 validate_task_id(task_id).map_err(|error| error.to_string())?;
544 let task = self
545 .task_for_session(task_id, session_id)
546 .ok_or_else(|| "task_not_found".to_string())?;
547 let mut file = open_task_artifact(&task.paths, artifact)
548 .map_err(|error| format!("artifact_refused: {error}"))?;
549 file.read_all()
550 .map_err(|error| format!("artifact_refused: {error}"))
551 }
552
553 pub fn set_harness(&self, harness: Harness) {
554 if let Ok(mut slot) = self.inner.db_harness.write() {
555 *slot = Some(harness.storage_segment());
556 }
557 }
558
559 pub fn set_db_pool(&self, conn: Arc<Mutex<TrackedConnection>>) {
560 if let Ok(mut slot) = self.inner.db_pool.write() {
561 *slot = Some(conn);
562 }
563 self.inner.compression_aggregates.clear();
564 }
565
566 pub fn clear_db_pool(&self) {
567 if let Ok(mut slot) = self.inner.db_pool.write() {
568 *slot = None;
569 }
570 self.inner.compression_aggregates.clear();
571 }
572
573 pub(crate) fn record_live_delivery_session(&self, session_id: &str) {
574 if let Ok(mut live_sessions) = self.inner.live_delivery_sessions.lock() {
575 live_sessions.insert(session_id.to_string());
576 }
577 }
578
579 pub(crate) fn replace_live_delivery_sessions(&self, sessions: HashSet<String>) {
580 if let Ok(mut live_sessions) = self.inner.live_delivery_sessions.lock() {
581 *live_sessions = sessions;
582 }
583 }
584
585 fn originating_session_has_live_route(&self, session_id: &str) -> bool {
586 self.inner
587 .live_delivery_sessions
588 .lock()
589 .map(|sessions| sessions.contains(session_id))
590 .unwrap_or(true)
591 }
592
593 fn should_retire_foreign_delivery(
594 &self,
595 originating_session_id: &str,
596 binding_session_id: &str,
597 ) -> bool {
598 originating_session_id != binding_session_id
599 && !self.originating_session_has_live_route(originating_session_id)
600 }
601
602 pub(crate) fn compression_aggregate_cache(&self) -> Arc<CompressionAggregateCache> {
603 Arc::clone(&self.inner.compression_aggregates)
604 }
605
606 pub fn register_foreground_task(&self, session_id: &str, task_id: &str) {
607 if let Ok(mut tasks) = self.inner.wait_registered_tasks.lock() {
608 tasks
609 .entry(session_id.to_string())
610 .or_default()
611 .insert(task_id.to_string());
612 }
613 }
614
615 pub fn begin_wait_mode_session(&self, session_id: &str, task_id: &str) {
616 if let Ok(mut active) = self.inner.active_wait_sessions.lock() {
617 *active.entry(session_id.to_string()).or_insert(0) += 1;
618 }
619 self.register_foreground_task(session_id, task_id);
620 if let Ok(mut detach) = self.inner.wait_detach_sessions.lock() {
621 detach.remove(session_id);
622 }
623 }
624
625 pub fn unregister_foreground_task(&self, session_id: &str, task_id: &str) {
626 if let Ok(mut tasks) = self.inner.wait_registered_tasks.lock() {
627 if let Some(session_tasks) = tasks.get_mut(session_id) {
628 session_tasks.remove(task_id);
629 if session_tasks.is_empty() {
630 tasks.remove(session_id);
631 }
632 }
633 }
634 }
635
636 pub fn end_wait_mode_session(&self, session_id: &str, task_id: &str) {
637 let no_active_wait = if let Ok(mut active) = self.inner.active_wait_sessions.lock() {
638 match active.get_mut(session_id) {
639 Some(count) if *count > 1 => *count -= 1,
640 Some(_) => {
641 active.remove(session_id);
642 }
643 None => {}
644 }
645 !active.contains_key(session_id)
646 } else {
647 false
648 };
649 self.unregister_foreground_task(session_id, task_id);
650 if no_active_wait {
651 if let Ok(mut detach) = self.inner.wait_detach_sessions.lock() {
652 detach.remove(session_id);
653 }
654 }
655 }
656
657 pub fn abort_inflight(&self, session_id: &str) -> Result<usize, String> {
661 let task_ids = self
662 .inner
663 .wait_registered_tasks
664 .lock()
665 .map(|mut tasks| tasks.remove(session_id).unwrap_or_default())
666 .map_err(|_| "wait registration lock poisoned".to_string())?;
667 if let Ok(mut active) = self.inner.active_wait_sessions.lock() {
668 active.remove(session_id);
669 }
670 if let Ok(mut detach) = self.inner.wait_detach_sessions.lock() {
671 detach.remove(session_id);
672 }
673
674 let mut killed = 0;
675 for task_id in task_ids {
676 let Some(task) = self.task_for_session(&task_id, session_id) else {
677 continue;
678 };
679 let is_terminal = task
680 .state
681 .lock()
682 .map(|state| state.metadata.status.is_terminal())
683 .map_err(|_| "background task lock poisoned".to_string())?;
684 if is_terminal {
685 continue;
686 }
687 let snapshot = self.kill_with_status_reason(
688 &task_id,
689 session_id,
690 BgTaskStatus::Killed,
691 Some("call_aborted".to_string()),
692 )?;
693 if snapshot.info.status == BgTaskStatus::Killed
694 && snapshot.info.status_reason.as_deref() == Some("call_aborted")
695 {
696 killed += 1;
697 }
698 }
699 Ok(killed)
700 }
701
702 pub fn signal_wait_mode_detach(&self, session_id: &str) -> bool {
703 let is_waiting = self
704 .inner
705 .active_wait_sessions
706 .lock()
707 .map(|active| active.get(session_id).copied().unwrap_or(0) > 0)
708 .unwrap_or(false);
709 if !is_waiting {
710 return false;
711 }
712 self.inner
713 .wait_detach_sessions
714 .lock()
715 .map(|mut detach| detach.insert(session_id.to_string()))
716 .unwrap_or(false)
717 }
718
719 pub fn active_wait_session_count(&self) -> usize {
722 self.inner
723 .active_wait_sessions
724 .lock()
725 .map(|active| active.len())
726 .unwrap_or(0)
727 }
728
729 pub fn take_wait_mode_detach(&self, session_id: &str) -> bool {
730 self.inner
731 .wait_detach_sessions
732 .lock()
733 .map(|mut detach| detach.remove(session_id))
734 .unwrap_or(false)
735 }
736
737 pub(crate) fn wait_mode_detach_pending(&self, session_id: &str) -> bool {
738 self.inner
739 .wait_detach_sessions
740 .lock()
741 .map(|detach| detach.contains(session_id))
742 .unwrap_or(false)
743 }
744
745 pub(crate) async fn terminal_transition_notified(&self) {
746 self.inner.terminal_transition.notified().await;
747 }
748
749 pub fn set_compressor<F>(&self, compressor: F)
754 where
755 F: Fn(&str, String) -> CompressionResult + Send + Sync + 'static,
756 {
757 self.set_compressor_with_exit_code(move |command, output, _exit_code| {
758 compressor(command, output)
759 });
760 }
761
762 pub fn set_compressor_with_exit_code<F>(&self, compressor: F)
763 where
764 F: Fn(&str, String, Option<i32>) -> CompressionResult + Send + Sync + 'static,
765 {
766 if let Ok(mut slot) = self.inner.compressor.lock() {
767 *slot = Some(Box::new(compressor));
768 }
769 }
770
771 pub(crate) fn compress_output(
774 &self,
775 command: &str,
776 output: String,
777 exit_code: Option<i32>,
778 ) -> CompressionResult {
779 let Ok(slot) = self.inner.compressor.lock() else {
780 return CompressionResult::new(output);
781 };
782 match slot.as_ref() {
783 Some(compressor) => compressor(command, output, exit_code),
784 None => CompressionResult::new(output),
785 }
786 }
787
788 fn ensure_terminal_output_cache(&self, task: &Arc<BgTask>) -> Option<TerminalOutputCache> {
789 let (metadata, buffer) = {
790 let state = task.state.lock().ok()?;
791 if !state.metadata.status.is_terminal() || state.metadata.mode == BgMode::Pty {
792 return None;
793 }
794 if let Some(cache) = state.terminal_output_cache.clone() {
795 return Some(cache);
796 }
797 (state.metadata.clone(), state.buffer.clone())
798 };
799
800 let mut cap_buffer = buffer.clone();
801 let disk_truncation = cap_buffer.enforce_terminal_cap();
802 let cache =
803 self.render_terminal_output(&metadata, &cap_buffer, disk_truncation, Some(&task.paths));
804 let mut state = task.state.lock().ok()?;
805 if !state.metadata.status.is_terminal() || state.metadata.mode == BgMode::Pty {
806 return None;
807 }
808 if let Some(existing) = state.terminal_output_cache.clone() {
809 return Some(existing);
810 }
811 state.terminal_output_cache = Some(cache.clone());
812 Some(cache)
813 }
814
815 fn render_terminal_output(
816 &self,
817 metadata: &PersistedTask,
818 buffer: &BgBuffer,
819 disk_truncation: DiskTruncation,
820 paths: Option<&TaskPaths>,
821 ) -> TerminalOutputCache {
822 let output_readable = buffer
823 .output_path()
824 .is_some_and(|path| self.is_session_owned_artifact_path(&metadata.session_id, &path));
825 let stderr_readable = buffer
826 .stderr_path()
827 .map(|path| self.is_session_owned_artifact_path(&metadata.session_id, path))
828 .unwrap_or(true);
829 let artifact_access = ArtifactRecoveryAccess {
830 task_id: metadata.task_id.clone(),
831 readable: output_readable && stderr_readable,
832 };
833
834 if metadata.mode == BgMode::Pty {
835 return TerminalOutputCache {
836 output_preview: String::new(),
837 output_truncated: false,
838 compression_input_line_count: None,
839 kind: TerminalOutputKind::Raw,
840 output_path: buffer.output_path().map(|path| path.display().to_string()),
841 stderr_path: buffer.stderr_path().map(|path| path.display().to_string()),
842 artifact_access,
843 recovery: None,
844 };
845 }
846
847 let mut rendered = if let Some(structured) = render_structured_output(
848 &metadata.command,
849 buffer,
850 disk_truncation,
851 artifact_access.clone(),
852 ) {
853 structured
854 } else if !metadata.compressed {
855 render_raw_passthrough(buffer, disk_truncation, artifact_access)
856 } else {
857 let raw = buffer.read_combined_head_tail(
858 COMPRESS_INPUT_CAP_BYTES,
859 COMPRESS_INPUT_HEAD_BYTES,
860 COMPRESS_INPUT_TAIL_BYTES,
861 );
862 let compressed = self.compress_output(&metadata.command, raw.text, metadata.exit_code);
863 render_compressed_with_recovery(
864 buffer,
865 compressed,
866 raw.truncated,
867 disk_truncation,
868 artifact_access,
869 )
870 };
871 normalize_piped_display_output(&mut rendered.output_preview);
872 append_pipeline_warning(&mut rendered, metadata, paths);
873 rendered
874 }
875
876 fn snapshot_with_terminal_cache(
877 &self,
878 task: &Arc<BgTask>,
879 preview_bytes: usize,
880 ) -> BgTaskSnapshot {
881 let mut snapshot = task.snapshot(preview_bytes);
882 self.maybe_compress_snapshot(task, &mut snapshot);
883 snapshot
884 }
885
886 fn post_terminal_transition(&self, task: &Arc<BgTask>, emit_frame: bool) -> Result<(), String> {
887 let should_sample = {
888 let mut state = task
889 .state
890 .lock()
891 .map_err(|_| "background task lock poisoned".to_string())?;
892 if !state.metadata.status.is_terminal() {
893 return Ok(());
894 }
895 if state.metadata.mode != BgMode::Pipes || state.descendant_sampling_started {
896 false
897 } else {
898 state.descendant_sampling_started = true;
899 true
900 }
901 };
902
903 let survivors_at_exit = should_sample
910 && self
911 .sample_task_process_group(task)
912 .is_some_and(|(members, omitted)| !members.is_empty() || omitted > 0);
913 if !survivors_at_exit {
914 let result = self.finish_terminal_transition(task, emit_frame);
915 self.inner.terminal_transition.notify_waiters();
916 return result;
917 }
918
919 let registry = self.clone();
923 let task = Arc::clone(task);
924 std::thread::Builder::new()
925 .name(format!("aft-bg-descendants-{}", task.task_id))
926 .spawn(move || {
927 std::thread::sleep(Duration::from_millis(300));
928 let settled = registry.sample_task_process_group(&task);
929 if let Err(error) = registry.finish_terminal_transition(&task, emit_frame) {
930 crate::slog_warn!(
931 "failed to finish background task {} after descendant sampling: {error}",
932 task.task_id
933 );
934 }
935 registry.inner.terminal_transition.notify_waiters();
936 if settled
937 .as_ref()
938 .is_some_and(|(members, omitted)| !members.is_empty() || *omitted > 0)
939 {
940 std::thread::sleep(Duration::from_millis(1_700));
941 let _ = registry.sample_task_process_group(&task);
942 }
943 })
944 .map_err(|error| format!("failed to start descendant sampler: {error}"))?;
945 Ok(())
946 }
947
948 fn sample_task_process_group(
949 &self,
950 task: &Arc<BgTask>,
951 ) -> Option<(Vec<LiveDescendant>, usize)> {
952 let pgid = task
953 .state
954 .lock()
955 .ok()
956 .and_then(|state| state.metadata.pgid)?;
957 let sample = live_process_group_members(pgid);
958 if let Ok(mut state) = task.state.lock() {
959 state.metadata.live_descendants = sample.as_ref().map(|(members, _)| members.clone());
960 state.metadata.live_descendants_omitted =
961 sample.as_ref().map(|(_, omitted)| *omitted).unwrap_or(0);
962 if let Err(error) = self.persist_task(&task.paths, &state.metadata) {
963 crate::slog_warn!(
964 "failed to persist live descendants for {}: {error}",
965 task.task_id
966 );
967 }
968 }
969 sample
970 }
971
972 fn finish_terminal_transition(
973 &self,
974 task: &Arc<BgTask>,
975 emit_frame: bool,
976 ) -> Result<(), String> {
977 let (metadata, buffer) = {
978 let state = task
979 .state
980 .lock()
981 .map_err(|_| "background task lock poisoned".to_string())?;
982 if !state.metadata.status.is_terminal() {
983 return Ok(());
984 }
985 (state.metadata.clone(), state.buffer.clone())
986 };
987
988 let cache = self.ensure_terminal_output_cache(task);
989 self.enqueue_completion_from_parts(
990 &metadata,
991 Some(&buffer),
992 None,
993 emit_frame,
994 cache.as_ref(),
995 );
996 Ok(())
997 }
998
999 fn persist_task(&self, paths: &TaskPaths, metadata: &PersistedTask) -> std::io::Result<()> {
1000 let task = resolve_task_layout(&paths.session_dir, &paths.task_id)?;
1001 write_task_at(&task, metadata)?;
1002 self.dual_write_task(paths, metadata);
1003 Ok(())
1004 }
1005
1006 fn update_task_metadata<F>(
1007 &self,
1008 paths: &TaskPaths,
1009 update: F,
1010 ) -> std::io::Result<PersistedTask>
1011 where
1012 F: FnOnce(&mut PersistedTask),
1013 {
1014 let task = resolve_task_layout(&paths.session_dir, &paths.task_id)?;
1015 let metadata = update_task_at(&task, update)?;
1016 self.dual_write_task(paths, &metadata);
1017 Ok(metadata)
1018 }
1019
1020 fn dual_write_task(&self, paths: &TaskPaths, metadata: &PersistedTask) {
1021 let pool = self.inner.db_pool.read().ok().and_then(|slot| slot.clone());
1022 let Some(pool) = pool else {
1023 return;
1024 };
1025 let harness = self
1026 .inner
1027 .db_harness
1028 .read()
1029 .ok()
1030 .and_then(|slot| slot.clone());
1031 let Some(harness) = harness else {
1032 crate::slog_warn!(
1033 "dual-write bash_task to DB skipped for {}: harness not configured",
1034 metadata.task_id
1035 );
1036 return;
1037 };
1038 let row = match metadata.to_bash_task_row(&harness, paths) {
1039 Ok(row) => row,
1040 Err(error) => {
1041 crate::slog_warn!(
1042 "dual-write bash_task to DB failed for {}: {}",
1043 metadata.task_id,
1044 error
1045 );
1046 return;
1047 }
1048 };
1049 let conn = match pool.lock() {
1050 Ok(conn) => conn,
1051 Err(_) => {
1052 crate::slog_warn!(
1053 "dual-write bash_task to DB failed for {}: db mutex poisoned",
1054 metadata.task_id
1055 );
1056 return;
1057 }
1058 };
1059 if let Err(error) = crate::db::bash_tasks::upsert_bash_task(&conn, &row) {
1060 crate::slog_warn!(
1061 "dual-write bash_task to DB failed for {}: {}",
1062 metadata.task_id,
1063 error
1064 );
1065 }
1066 }
1067
1068 fn delete_gc_task_from_db(&self, metadata: &PersistedTask) {
1069 let pool = self.inner.db_pool.read().ok().and_then(|slot| slot.clone());
1070 let Some(pool) = pool else {
1071 return;
1072 };
1073 let harness = self
1074 .inner
1075 .db_harness
1076 .read()
1077 .ok()
1078 .and_then(|slot| slot.clone());
1079 let Some(harness) = harness else {
1080 crate::slog_warn!(
1081 "GC bash_task DB delete skipped for {}: harness not configured",
1082 metadata.task_id
1083 );
1084 return;
1085 };
1086 let conn = match pool.lock() {
1087 Ok(conn) => conn,
1088 Err(_) => {
1089 crate::slog_warn!(
1090 "GC bash_task DB delete failed for {}: db mutex poisoned",
1091 metadata.task_id
1092 );
1093 return;
1094 }
1095 };
1096 if let Err(error) = crate::db::bash_tasks::delete_delivered_terminal_bash_task(
1097 &conn,
1098 &harness,
1099 &metadata.session_id,
1100 &metadata.task_id,
1101 "persisted_gc_delivered_terminal",
1102 ) {
1103 crate::slog_warn!(
1104 "GC bash_task DB delete failed for {}: {}",
1105 metadata.task_id,
1106 error
1107 );
1108 }
1109 }
1110
1111 fn persisted_task_process_is_alive(metadata: &PersistedTask) -> bool {
1112 let child_pid = metadata.child_pid;
1113 let group_leader = metadata.pgid.and_then(|pid| u32::try_from(pid).ok());
1114 child_pid
1115 .into_iter()
1116 .chain(group_leader)
1117 .any(|pid| is_recorded_process_alive(pid, metadata.started_at))
1118 }
1119
1120 fn db_has_live_process_for_task(&self, task_id: &str) -> bool {
1121 let Some((harness, pool)) = self.db_harness_and_pool() else {
1122 return false;
1123 };
1124 let Ok(conn) = pool.lock() else {
1125 return false;
1126 };
1127 crate::db::bash_tasks::list_bash_tasks_by_id(&conn, &harness, task_id)
1128 .map(|rows| {
1129 rows.into_iter().any(|row| {
1130 let started_at = u64::try_from(row.started_at).unwrap_or_default();
1131 row.pid
1132 .and_then(|pid| u32::try_from(pid).ok())
1133 .into_iter()
1134 .chain(row.pgid.and_then(|pid| u32::try_from(pid).ok()))
1135 .any(|pid| is_recorded_process_alive(pid, started_at))
1136 })
1137 })
1138 .unwrap_or(false)
1139 }
1140
1141 fn db_harness_and_pool(&self) -> Option<(String, Arc<Mutex<TrackedConnection>>)> {
1142 let pool = self
1143 .inner
1144 .db_pool
1145 .read()
1146 .ok()
1147 .and_then(|slot| slot.clone())?;
1148 let harness = self
1149 .inner
1150 .db_harness
1151 .read()
1152 .ok()
1153 .and_then(|slot| slot.clone())?;
1154 Some((harness, pool))
1155 }
1156
1157 pub fn pending_pattern_matches_for_session(
1158 &self,
1159 session_id: &str,
1160 ) -> Vec<BashPatternMatchFrame> {
1161 let Some((harness, pool)) = self.db_harness_and_pool() else {
1162 return Vec::new();
1163 };
1164 let rows = {
1165 let Ok(conn) = pool.lock() else {
1166 return Vec::new();
1167 };
1168 match crate::db::bash_watches::list_bash_pattern_watches_for_session(
1169 &conn, &harness, session_id,
1170 ) {
1171 Ok(rows) => rows,
1172 Err(error) => {
1173 crate::slog_warn!(
1174 "failed to load pending bash watches for session {session_id}: {error}"
1175 );
1176 return Vec::new();
1177 }
1178 }
1179 };
1180
1181 rows.into_iter()
1182 .filter(|row| row.pending_match)
1183 .filter_map(|row| {
1184 let Some(match_text) = row.match_text else {
1185 crate::slog_warn!(
1186 "pending bash watch {}/{} has no match text",
1187 row.task_id,
1188 row.watch_id
1189 );
1190 return None;
1191 };
1192 let context = row.match_context.unwrap_or_else(|| match_text.clone());
1193 Some(if match_text == WATCH_TARGET_ERASED_TEXT {
1194 BashPatternMatchFrame::watch_target_erased(
1195 row.task_id,
1196 session_id,
1197 row.watch_id,
1198 match_text,
1199 context,
1200 )
1201 } else if match_text == WATCH_TASK_EXIT_TEXT {
1202 BashPatternMatchFrame::task_exit(row.task_id, session_id, match_text, context)
1203 } else {
1204 BashPatternMatchFrame::new(
1205 row.task_id,
1206 session_id,
1207 row.watch_id,
1208 match_text,
1209 row.match_offset.unwrap_or_default().max(0) as u64,
1210 context,
1211 row.once,
1212 )
1213 })
1214 })
1215 .collect()
1216 }
1217
1218 fn terminal_db_status_for_session(
1219 &self,
1220 session_id: &str,
1221 task_id: &str,
1222 storage_dir: &Path,
1223 ) -> Option<BgTaskSnapshot> {
1224 let (harness, pool) = self.db_harness_and_pool()?;
1225 let conn = pool.lock().ok()?;
1226 let row =
1227 crate::db::bash_tasks::get_bash_task(&conn, &harness, session_id, task_id).ok()??;
1228 if !task_bundle_is_absent(storage_dir, &row.session_id, &row.task_id) {
1229 return None;
1230 }
1231 let metadata = PersistedTask::from(row.clone());
1232 metadata
1233 .status
1234 .is_terminal()
1235 .then(|| terminal_db_row_snapshot(row, metadata))
1236 }
1237
1238 pub fn has_erased_watch_reference(&self, task_id: &str) -> bool {
1239 self.evaluate_erased_watch_targets();
1240 self.inner
1241 .watch_registry
1242 .lock()
1243 .map(|registry| registry.has_erased_task(task_id))
1244 .unwrap_or(false)
1245 }
1246
1247 pub(crate) fn evaluate_erased_watch_targets(&self) {
1248 let Some((harness, pool)) = self.db_harness_and_pool() else {
1249 return;
1250 };
1251 let watched_task_ids = self
1254 .inner
1255 .watch_registry
1256 .lock()
1257 .map(|registry| registry.watched_task_ids())
1258 .unwrap_or_default();
1259 if watched_task_ids.is_empty() {
1260 return;
1261 }
1262
1263 let candidates = watched_task_ids
1264 .into_iter()
1265 .filter_map(|task_id| {
1266 let task = self.task(&task_id)?;
1267 self.originating_session_has_live_route(&task.session_id)
1268 .then(|| (task.session_id.clone(), task_id))
1269 })
1270 .collect::<Vec<_>>();
1271 let erased_tasks = {
1272 let Ok(conn) = pool.lock() else {
1273 return;
1274 };
1275 let mut erased_tasks = Vec::new();
1276 for (session_id, task_id) in candidates {
1277 match crate::db::bash_tasks::get_bash_task(&conn, &harness, &session_id, &task_id) {
1278 Ok(None) => {
1279 if let Err(error) =
1280 crate::db::bash_watches::delete_bash_pattern_watches_for_task(
1281 &conn,
1282 &harness,
1283 &session_id,
1284 &task_id,
1285 )
1286 {
1287 crate::slog_warn!(
1288 "failed to retire durable watches for erased task {}: {error}",
1289 task_id
1290 );
1291 }
1292 erased_tasks.push((session_id, task_id));
1293 }
1294 Ok(Some(_)) => {}
1295 Err(error) => {
1296 crate::slog_warn!(
1297 "failed to inspect bash watch target {}: {error}",
1298 task_id
1299 );
1300 }
1301 }
1302 }
1303 erased_tasks
1304 };
1305
1306 for (session_id, task_id) in erased_tasks {
1307 let watch_ids = self
1308 .inner
1309 .watch_registry
1310 .lock()
1311 .map(|mut registry| registry.terminalize_erased_task(&task_id))
1312 .unwrap_or_default();
1313 for watch_id in watch_ids {
1314 self.emit_bash_watch_erased(&session_id, &task_id, &watch_id);
1315 }
1316 }
1317 }
1318
1319 fn persist_watch_registration(
1320 &self,
1321 session_id: &str,
1322 task_id: &str,
1323 watch_id: &str,
1324 pattern: &WatchPattern,
1325 once: bool,
1326 stdout_offset: u64,
1327 stderr_offset: u64,
1328 pty_offset: u64,
1329 ) {
1330 let Some((harness, pool)) = self.db_harness_and_pool() else {
1331 return;
1332 };
1333 let Ok(conn) = pool.lock() else {
1334 return;
1335 };
1336 let row = BashPatternWatchRow {
1337 harness,
1338 session_id: session_id.to_string(),
1339 task_id: task_id.to_string(),
1340 watch_id: watch_id.to_string(),
1341 pattern_kind: pattern.kind_name().to_string(),
1342 pattern: pattern.pattern_text().to_string(),
1343 once,
1344 created_at: unix_millis() as i64,
1345 stdout_offset: stdout_offset as i64,
1346 stderr_offset: stderr_offset as i64,
1347 pty_offset: pty_offset as i64,
1348 scanning: true,
1349 pending_match: false,
1350 match_text: None,
1351 match_offset: None,
1352 match_context: None,
1353 };
1354 if let Err(error) = crate::db::bash_watches::upsert_bash_pattern_watch(&conn, &row) {
1355 crate::slog_warn!(
1356 "persist bash_pattern_watch failed for {task_id}/{watch_id}: {error}"
1357 );
1358 }
1359 }
1360
1361 fn delete_persisted_watch(&self, session_id: &str, task_id: &str, watch_id: &str) {
1362 let Some((harness, pool)) = self.db_harness_and_pool() else {
1363 return;
1364 };
1365 let Ok(conn) = pool.lock() else {
1366 return;
1367 };
1368 if let Err(error) = crate::db::bash_watches::delete_bash_pattern_watch(
1369 &conn, &harness, session_id, task_id, watch_id,
1370 ) {
1371 crate::slog_warn!("delete bash_pattern_watch failed for {task_id}/{watch_id}: {error}");
1372 }
1373 }
1374
1375 fn delete_persisted_watches_for_task(&self, session_id: &str, task_id: &str) {
1376 let Some((harness, pool)) = self.db_harness_and_pool() else {
1377 return;
1378 };
1379 let Ok(conn) = pool.lock() else {
1380 return;
1381 };
1382 if let Err(error) = crate::db::bash_watches::delete_bash_pattern_watches_for_task(
1383 &conn, &harness, session_id, task_id,
1384 ) {
1385 crate::slog_warn!("delete bash_pattern_watches for {task_id} failed: {error}");
1386 }
1387 }
1388
1389 fn persist_watch_match(
1390 &self,
1391 session_id: &str,
1392 task_id: &str,
1393 pattern_match: &PatternMatch,
1394 stdout_offset: u64,
1395 stderr_offset: u64,
1396 pty_offset: u64,
1397 ) {
1398 let Some((harness, pool)) = self.db_harness_and_pool() else {
1399 return;
1400 };
1401 let Ok(conn) = pool.lock() else {
1402 return;
1403 };
1404 let Ok(Some(mut row)) = crate::db::bash_watches::get_bash_pattern_watch(
1405 &conn,
1406 &harness,
1407 session_id,
1408 task_id,
1409 &pattern_match.watch_id,
1410 ) else {
1411 return;
1412 };
1413 row.stdout_offset = stdout_offset as i64;
1414 row.stderr_offset = stderr_offset as i64;
1415 row.pty_offset = pty_offset as i64;
1416 row.pending_match = true;
1417 row.match_text = Some(pattern_match.match_text.clone());
1418 row.match_offset = Some(pattern_match.match_offset as i64);
1419 row.match_context = Some(pattern_match.context.clone());
1420 if pattern_match.once {
1421 row.scanning = false;
1424 }
1425 if let Err(error) = crate::db::bash_watches::upsert_bash_pattern_watch(&conn, &row) {
1426 crate::slog_warn!(
1427 "persist bash_pattern_watch match failed for {}/{}: {error}",
1428 task_id,
1429 pattern_match.watch_id
1430 );
1431 }
1432 }
1433
1434 fn persist_task_watch_cursors(
1435 &self,
1436 session_id: &str,
1437 task_id: &str,
1438 stdout_offset: u64,
1439 stderr_offset: u64,
1440 pty_offset: u64,
1441 ) {
1442 let Some((harness, pool)) = self.db_harness_and_pool() else {
1443 return;
1444 };
1445 let Ok(conn) = pool.lock() else {
1446 return;
1447 };
1448 if let Err(error) = crate::db::bash_watches::update_watch_offsets_for_task(
1449 &conn,
1450 &harness,
1451 session_id,
1452 task_id,
1453 stdout_offset as i64,
1454 stderr_offset as i64,
1455 pty_offset as i64,
1456 ) {
1457 crate::slog_warn!("persist bash_pattern_watch cursors failed for {task_id}: {error}");
1458 }
1459 }
1460
1461 fn watch_stream_cursors(&self, task_id: &str) -> (u64, u64, u64) {
1462 let Ok(registry) = self.inner.watch_registry.lock() else {
1463 return (0, 0, 0);
1464 };
1465 let stdout = registry
1466 .file_cursor(&format!("{task_id}:stdout"))
1467 .unwrap_or(0);
1468 let stderr = registry
1469 .file_cursor(&format!("{task_id}:stderr"))
1470 .unwrap_or(0);
1471 let pty = registry.file_cursor(&format!("{task_id}:pty")).unwrap_or(0);
1472 (stdout, stderr, pty)
1473 }
1474
1475 fn ack_persisted_watches_for_task(&self, session_id: &str, task_id: &str, task_terminal: bool) {
1478 let Some((harness, pool)) = self.db_harness_and_pool() else {
1479 return;
1480 };
1481 let Ok(conn) = pool.lock() else {
1482 return;
1483 };
1484 if task_terminal {
1485 let _ = crate::db::bash_watches::delete_bash_pattern_watches_for_task(
1486 &conn, &harness, session_id, task_id,
1487 );
1488 return;
1489 }
1490 let Ok(rows) = crate::db::bash_watches::list_bash_pattern_watches_for_task(
1491 &conn, &harness, session_id, task_id,
1492 ) else {
1493 return;
1494 };
1495 for mut row in rows {
1496 if row.once && (!row.scanning || row.pending_match) {
1497 let _ = crate::db::bash_watches::delete_bash_pattern_watch(
1498 &conn,
1499 &harness,
1500 session_id,
1501 task_id,
1502 &row.watch_id,
1503 );
1504 continue;
1505 }
1506 if row.pending_match {
1507 row.pending_match = false;
1508 row.match_text = None;
1509 row.match_offset = None;
1510 row.match_context = None;
1511 let _ = crate::db::bash_watches::upsert_bash_pattern_watch(&conn, &row);
1512 }
1513 }
1514 }
1515
1516 pub fn record_scanner_report(
1517 &self,
1518 task_id: &str,
1519 session_id: &str,
1520 scanner_report: Vec<PermissionAsk>,
1521 ) -> Result<(), String> {
1522 if scanner_report.is_empty() {
1523 return Ok(());
1524 }
1525 let task = self.task_for_session(task_id, session_id).ok_or_else(|| {
1526 "background task not found while recording scanner report".to_string()
1527 })?;
1528 let metadata = {
1529 let mut state = task
1530 .state
1531 .lock()
1532 .map_err(|_| "background task lock poisoned".to_string())?;
1533 state.metadata.scanner_report = scanner_report;
1534 state.metadata.clone()
1535 };
1536 self.persist_task(&task.paths, &metadata)
1537 .map_err(|error| format!("failed to persist scanner report: {error}"))
1538 }
1539
1540 pub fn configure_long_running_reminders(&self, enabled: bool, interval_ms: u64) {
1541 self.inner
1542 .long_running_reminder_enabled
1543 .store(enabled, Ordering::SeqCst);
1544 self.inner
1545 .long_running_reminder_interval_ms
1546 .store(interval_ms, Ordering::SeqCst);
1547 }
1548
1549 #[cfg(unix)]
1550 #[allow(clippy::too_many_arguments)]
1551 pub fn spawn(
1552 &self,
1553 spawn_plan: SpawnPlan,
1554 command: &str,
1555 session_id: String,
1556 workdir: PathBuf,
1557 env: HashMap<String, String>,
1558 timeout: Option<Duration>,
1559 storage_dir: PathBuf,
1560 max_running: usize,
1561 notify_on_completion: bool,
1562 compressed: bool,
1563 project_root: Option<PathBuf>,
1564 ) -> Result<String, String> {
1565 self.spawn_with_shell(
1566 spawn_plan,
1567 command,
1568 super::BashShell::Bash,
1569 resolve_posix_shell(),
1570 session_id,
1571 workdir,
1572 env,
1573 timeout,
1574 storage_dir,
1575 max_running,
1576 notify_on_completion,
1577 compressed,
1578 project_root,
1579 )
1580 }
1581
1582 #[cfg(unix)]
1583 #[allow(clippy::too_many_arguments)]
1584 pub fn spawn_with_shell(
1585 &self,
1586 spawn_plan: SpawnPlan,
1587 command: &str,
1588 shell: super::BashShell,
1589 shell_path: PathBuf,
1590 session_id: String,
1591 workdir: PathBuf,
1592 #[cfg_attr(not(target_os = "linux"), allow(unused_mut))] mut env: HashMap<String, String>,
1593 timeout: Option<Duration>,
1594 storage_dir: PathBuf,
1595 max_running: usize,
1596 notify_on_completion: bool,
1597 compressed: bool,
1598 project_root: Option<PathBuf>,
1599 ) -> Result<String, String> {
1600 self.start_watchdog();
1601
1602 #[cfg(target_os = "linux")]
1603 let linux_scope = env.remove(LINUX_SCOPE_ENV).as_deref() == Some("1");
1604 #[cfg(not(target_os = "linux"))]
1605 let linux_scope = false;
1606
1607 let running = self.running_count();
1608 if running >= max_running {
1609 #[cfg(unix)]
1610 if let Some(prepared) = spawn_plan.prepared_task() {
1611 let _ = delete_resolved_task(&prepared.resolved_task());
1612 }
1613 return Err(format!(
1614 "background bash task limit exceeded: {running} running (max {max_running})"
1615 ));
1616 }
1617
1618 let timeout = timeout.or(Some(DEFAULT_BG_TIMEOUT));
1619 let timeout_ms = timeout.map(|timeout| timeout.as_millis() as u64);
1620 let (spawn_plan, task_layout) = if let Some(prepared) = spawn_plan.prepared_task() {
1621 (spawn_plan.clone(), prepared.resolved_task())
1622 } else {
1623 let task = allocate_task_layout(&storage_dir, &session_id)
1624 .map_err(|error| format!("failed to create background task layout: {error}"))?;
1625 let root = project_root.as_deref().unwrap_or(&workdir);
1626 let environment =
1627 crate::sandbox_spawn::approved_payload_environment(&env, &std::env::temp_dir());
1628 let prepared = match crate::sandbox_spawn::prepare_task_payload(
1629 &task,
1630 command.as_bytes(),
1631 root,
1632 &workdir,
1633 &crate::sandbox_spawn::AuthenticatedPrincipal::FirstParty,
1634 &shell_path,
1635 &environment,
1636 ) {
1637 Ok(prepared) => prepared,
1638 Err(error) => {
1639 let _ = delete_resolved_task(&task);
1640 return Err(error);
1641 }
1642 };
1643 let task = prepared.resolved_task();
1644 (spawn_plan.with_prepared_task(prepared), task)
1645 };
1646 let task_id = task_layout.paths.task_id.clone();
1647 let paths = task_layout.paths.clone();
1648
1649 if self.task(&task_id).is_some() {
1650 let _ = delete_resolved_task(&task_layout);
1651 return Err("background task id collided with a live task".to_string());
1652 }
1653
1654 let mut metadata = PersistedTask::starting(
1655 task_id.clone(),
1656 session_id.clone(),
1657 command.to_string(),
1658 workdir.clone(),
1659 project_root,
1660 timeout_ms,
1661 notify_on_completion,
1662 compressed,
1663 );
1664 #[cfg(unix)]
1668 let capture_pipeline_status = {
1669 let pipeline = single_top_level_pipeline(command);
1670 let has_pipeline = pipeline.is_some();
1671 let capture = !shell.is_powershell()
1672 && should_capture_pipeline_status(&spawn_plan, has_pipeline, &shell_path);
1673 if !shell.is_powershell() {
1674 metadata.pipeline_segments = pipeline
1675 .as_ref()
1676 .map(|pipeline| {
1677 pipeline
1678 .segments
1679 .iter()
1680 .map(|segment| segment.label.clone())
1681 .collect()
1682 })
1683 .unwrap_or_default();
1684 if has_pipeline && !capture {
1685 metadata.pipeline_status_unavailable =
1686 Some(if spawn_plan.is_native_launcher() {
1687 "native sandbox launcher".to_string()
1688 } else {
1689 shell_path
1690 .file_name()
1691 .and_then(|name| name.to_str())
1692 .unwrap_or("unknown shell")
1693 .to_string()
1694 });
1695 }
1696 }
1697 capture
1698 };
1699 #[cfg(windows)]
1700 let capture_pipeline_status = false;
1701 attach_sandbox_metadata(&mut metadata, &spawn_plan);
1702 if let Err(error) = write_task_at(&task_layout, &metadata) {
1703 let _ = delete_resolved_task(&task_layout);
1704 return Err(format!(
1705 "failed to persist background task metadata: {error}"
1706 ));
1707 }
1708 self.dual_write_task(&paths, &metadata);
1709
1710 let mut io_handles =
1711 TaskIoHandles::create(&task_layout, BgMode::Pipes, capture_pipeline_status)
1712 .map_err(|error| format!("failed to pre-open task output handles: {error}"))?;
1713 let child = match spawn_detached_child(
1714 &spawn_plan,
1715 command,
1716 shell,
1717 &shell_path,
1718 &paths,
1719 &workdir,
1720 &env,
1721 &mut io_handles,
1722 capture_pipeline_status,
1723 linux_scope,
1724 ) {
1725 Ok(child) => child,
1726 Err(error) => {
1727 crate::slog_warn!("failed to spawn background bash task {task_id}; deleting partial bundle: {error}");
1728 let _ = delete_task_bundle(&paths);
1729 return Err(error);
1730 }
1731 };
1732
1733 let child_pid = child.id();
1734 metadata.mark_running(child_pid, child_pid as i32);
1735 self.persist_task(&paths, &metadata)
1736 .map_err(|e| format!("failed to persist running background task metadata: {e}"))?;
1737
1738 let task = Arc::new(BgTask {
1739 task_id: task_id.clone(),
1740 session_id,
1741 paths: paths.clone(),
1742 artifact_root: canonical_artifact_root(&paths),
1743 started: Instant::now(),
1744 last_reminder_at: Mutex::new(None),
1745 terminal_at: Mutex::new(None),
1746 state: Mutex::new(BgTaskState {
1747 metadata,
1748 runtime: TaskRuntime::Piped(Some(child)),
1749 io_handles: Some(io_handles),
1750 detached: false,
1751 child_exit_observed: false,
1752 descendant_sampling_started: false,
1753 buffer: BgBuffer::registered(&paths, BgMode::Pipes),
1754 terminal_output_cache: None,
1755 pending_terminal_override: None,
1756 }),
1757 });
1758
1759 self.record_live_delivery_session(&task.session_id);
1760 self.inner
1761 .tasks
1762 .lock()
1763 .map_err(|_| "background task registry lock poisoned".to_string())?
1764 .insert(task_id.clone(), task);
1765
1766 Ok(task_id)
1767 }
1768
1769 #[allow(clippy::too_many_arguments)]
1770 pub fn spawn_pty(
1771 &self,
1772 spawn_plan: SpawnPlan,
1773 command: &str,
1774 session_id: String,
1775 workdir: PathBuf,
1776 env: HashMap<String, String>,
1777 timeout: Option<Duration>,
1778 storage_dir: PathBuf,
1779 max_running: usize,
1780 notify_on_completion: bool,
1781 compressed: bool,
1782 project_root: Option<PathBuf>,
1783 rows: u16,
1784 cols: u16,
1785 ) -> Result<String, String> {
1786 self.spawn_pty_with_shell(
1787 spawn_plan,
1788 command,
1789 super::BashShell::Bash,
1790 super::resolve_shell_path(true, super::BashShell::Bash)
1791 .expect("POSIX shell must resolve for bash PTY"),
1792 session_id,
1793 workdir,
1794 env,
1795 timeout,
1796 storage_dir,
1797 max_running,
1798 notify_on_completion,
1799 compressed,
1800 project_root,
1801 rows,
1802 cols,
1803 )
1804 }
1805
1806 #[allow(clippy::too_many_arguments)]
1807 pub fn spawn_pty_with_shell(
1808 &self,
1809 spawn_plan: SpawnPlan,
1810 command: &str,
1811 shell: super::BashShell,
1812 shell_path: PathBuf,
1813 session_id: String,
1814 workdir: PathBuf,
1815 env: HashMap<String, String>,
1816 timeout: Option<Duration>,
1817 storage_dir: PathBuf,
1818 max_running: usize,
1819 notify_on_completion: bool,
1820 compressed: bool,
1821 project_root: Option<PathBuf>,
1822 rows: u16,
1823 cols: u16,
1824 ) -> Result<String, String> {
1825 self.start_watchdog();
1826
1827 let running = self.running_count();
1828 if running >= max_running {
1829 #[cfg(unix)]
1830 if let Some(prepared) = spawn_plan.prepared_task() {
1831 let _ = delete_resolved_task(&prepared.resolved_task());
1832 }
1833 return Err(format!(
1834 "background bash task limit exceeded: {running} running (max {max_running})"
1835 ));
1836 }
1837
1838 let timeout = timeout.or(Some(DEFAULT_BG_TIMEOUT));
1839 let timeout_ms = timeout.map(|timeout| timeout.as_millis() as u64);
1840 #[cfg(unix)]
1841 let (spawn_plan, task_layout) = if let Some(prepared) = spawn_plan.prepared_task() {
1842 (spawn_plan.clone(), prepared.resolved_task())
1843 } else {
1844 let task = allocate_task_layout(&storage_dir, &session_id)
1845 .map_err(|error| format!("failed to create PTY task layout: {error}"))?;
1846 let root = project_root.as_deref().unwrap_or(&workdir);
1847 let environment =
1848 crate::sandbox_spawn::approved_payload_environment(&env, &std::env::temp_dir());
1849 let prepared = match crate::sandbox_spawn::prepare_task_payload(
1850 &task,
1851 command.as_bytes(),
1852 root,
1853 &workdir,
1854 &crate::sandbox_spawn::AuthenticatedPrincipal::FirstParty,
1855 &shell_path,
1856 &environment,
1857 ) {
1858 Ok(prepared) => prepared,
1859 Err(error) => {
1860 let _ = delete_resolved_task(&task);
1861 return Err(error);
1862 }
1863 };
1864 let task = prepared.resolved_task();
1865 (spawn_plan.with_prepared_task(prepared), task)
1866 };
1867 #[cfg(windows)]
1868 let task_layout = allocate_task_layout(&storage_dir, &session_id)
1869 .map_err(|error| format!("failed to create PTY task layout: {error}"))?;
1870 let task_id = task_layout.paths.task_id.clone();
1871 let paths = task_layout.paths.clone();
1872
1873 let mut metadata = PersistedTask::starting(
1874 task_id.clone(),
1875 session_id.clone(),
1876 command.to_string(),
1877 workdir.clone(),
1878 project_root,
1879 timeout_ms,
1880 notify_on_completion,
1881 compressed,
1882 );
1883 attach_sandbox_metadata(&mut metadata, &spawn_plan);
1884 metadata.mode = BgMode::Pty;
1885 metadata.pty_rows = Some(rows);
1886 metadata.pty_cols = Some(cols);
1887 if let Err(error) = write_task_at(&task_layout, &metadata) {
1888 let _ = delete_resolved_task(&task_layout);
1889 return Err(format!(
1890 "failed to persist background task metadata: {error}"
1891 ));
1892 }
1893 self.dual_write_task(&paths, &metadata);
1894 let mut io_handles = TaskIoHandles::create(&task_layout, BgMode::Pty, false)
1895 .map_err(|error| format!("failed to pre-open PTY output handles: {error}"))?;
1896
1897 let runtime = match spawn_pty_for_command(
1898 &spawn_plan,
1899 &task_id,
1900 &session_id,
1901 command,
1902 shell,
1903 &shell_path,
1904 &paths,
1905 &workdir,
1906 &env,
1907 rows,
1908 cols,
1909 self.inner.wake_tx.clone(),
1910 &mut io_handles,
1911 ) {
1912 Ok(runtime) => runtime,
1913 Err(error) => {
1914 crate::slog_warn!(
1915 "failed to spawn PTY background bash task {task_id}; deleting partial bundle: {error}"
1916 );
1917 let _ = delete_task_bundle(&paths);
1918 return Err(error);
1919 }
1920 };
1921
1922 if let Some(child_pid) = runtime.child_pid {
1923 metadata.mark_running(child_pid, child_pid as i32);
1924 } else {
1925 metadata.status = BgTaskStatus::Running;
1926 metadata.pgid = None;
1927 }
1928 self.persist_task(&paths, &metadata)
1929 .map_err(|e| format!("failed to persist running background task metadata: {e}"))?;
1930
1931 let task = Arc::new(BgTask {
1932 task_id: task_id.clone(),
1933 session_id,
1934 paths: paths.clone(),
1935 artifact_root: canonical_artifact_root(&paths),
1936 started: Instant::now(),
1937 last_reminder_at: Mutex::new(None),
1938 terminal_at: Mutex::new(None),
1939 state: Mutex::new(BgTaskState {
1940 metadata,
1941 runtime: TaskRuntime::Pty(Some(runtime)),
1942 io_handles: Some(io_handles),
1943 detached: false,
1944 child_exit_observed: false,
1945 descendant_sampling_started: false,
1946 buffer: BgBuffer::registered(&paths, BgMode::Pty),
1947 terminal_output_cache: None,
1948 pending_terminal_override: None,
1949 }),
1950 });
1951
1952 self.record_live_delivery_session(&task.session_id);
1953 self.inner
1954 .tasks
1955 .lock()
1956 .map_err(|_| "background task registry lock poisoned".to_string())?
1957 .insert(task_id.clone(), task);
1958
1959 Ok(task_id)
1960 }
1961
1962 #[cfg(windows)]
1963 #[allow(clippy::too_many_arguments)]
1964 pub fn spawn(
1965 &self,
1966 spawn_plan: SpawnPlan,
1967 command: &str,
1968 session_id: String,
1969 workdir: PathBuf,
1970 env: HashMap<String, String>,
1971 timeout: Option<Duration>,
1972 storage_dir: PathBuf,
1973 max_running: usize,
1974 notify_on_completion: bool,
1975 compressed: bool,
1976 project_root: Option<PathBuf>,
1977 ) -> Result<String, String> {
1978 self.spawn_with_shell(
1979 spawn_plan,
1980 command,
1981 super::BashShell::Bash,
1982 PathBuf::from("cmd.exe"),
1983 session_id,
1984 workdir,
1985 env,
1986 timeout,
1987 storage_dir,
1988 max_running,
1989 notify_on_completion,
1990 compressed,
1991 project_root,
1992 )
1993 }
1994
1995 #[cfg(windows)]
1996 #[allow(clippy::too_many_arguments)]
1997 pub fn spawn_with_shell(
1998 &self,
1999 spawn_plan: SpawnPlan,
2000 command: &str,
2001 shell: super::BashShell,
2002 shell_path: PathBuf,
2003 session_id: String,
2004 workdir: PathBuf,
2005 env: HashMap<String, String>,
2006 timeout: Option<Duration>,
2007 storage_dir: PathBuf,
2008 max_running: usize,
2009 notify_on_completion: bool,
2010 compressed: bool,
2011 project_root: Option<PathBuf>,
2012 ) -> Result<String, String> {
2013 self.start_watchdog();
2014
2015 let running = self.running_count();
2016 if running >= max_running {
2017 #[cfg(unix)]
2018 if let Some(prepared) = spawn_plan.prepared_task() {
2019 let _ = delete_resolved_task(&prepared.resolved_task());
2020 }
2021 return Err(format!(
2022 "background bash task limit exceeded: {running} running (max {max_running})"
2023 ));
2024 }
2025
2026 let timeout = timeout.or(Some(DEFAULT_BG_TIMEOUT));
2027 let timeout_ms = timeout.map(|timeout| timeout.as_millis() as u64);
2028 let task_layout = allocate_task_layout(&storage_dir, &session_id)
2029 .map_err(|error| format!("failed to create background task layout: {error}"))?;
2030 let task_id = task_layout.paths.task_id.clone();
2031 let paths = task_layout.paths.clone();
2032
2033 let mut metadata = PersistedTask::starting(
2034 task_id.clone(),
2035 session_id.clone(),
2036 command.to_string(),
2037 workdir.clone(),
2038 project_root,
2039 timeout_ms,
2040 notify_on_completion,
2041 compressed,
2042 );
2043 attach_sandbox_metadata(&mut metadata, &spawn_plan);
2044 if let Err(error) = write_task_at(&task_layout, &metadata) {
2045 let _ = delete_resolved_task(&task_layout);
2046 return Err(format!(
2047 "failed to persist background task metadata: {error}"
2048 ));
2049 }
2050 self.dual_write_task(&paths, &metadata);
2051 let mut io_handles = TaskIoHandles::create(&task_layout, BgMode::Pipes, false)
2052 .map_err(|error| format!("failed to pre-open task output handles: {error}"))?;
2053
2054 let child = match spawn_detached_child(
2055 &spawn_plan,
2056 command,
2057 shell,
2058 &shell_path,
2059 &paths,
2060 &workdir,
2061 &env,
2062 &mut io_handles,
2063 false,
2064 false,
2065 ) {
2066 Ok(child) => child,
2067 Err(error) => {
2068 crate::slog_warn!("failed to spawn background bash task {task_id}; deleting partial bundle: {error}");
2069 let _ = delete_task_bundle(&paths);
2070 return Err(error);
2071 }
2072 };
2073
2074 let child_pid = child.id();
2075 metadata.status = BgTaskStatus::Running;
2076 metadata.child_pid = Some(child_pid);
2077 metadata.pgid = None;
2078 self.persist_task(&paths, &metadata)
2079 .map_err(|e| format!("failed to persist running background task metadata: {e}"))?;
2080
2081 let task = Arc::new(BgTask {
2082 task_id: task_id.clone(),
2083 session_id,
2084 paths: paths.clone(),
2085 artifact_root: canonical_artifact_root(&paths),
2086 started: Instant::now(),
2087 last_reminder_at: Mutex::new(None),
2088 terminal_at: Mutex::new(None),
2089 state: Mutex::new(BgTaskState {
2090 metadata,
2091 runtime: TaskRuntime::Piped(Some(child)),
2092 io_handles: Some(io_handles),
2093 detached: false,
2094 child_exit_observed: false,
2095 descendant_sampling_started: false,
2096 buffer: BgBuffer::registered(&paths, BgMode::Pipes),
2097 terminal_output_cache: None,
2098 pending_terminal_override: None,
2099 }),
2100 });
2101
2102 self.record_live_delivery_session(&task.session_id);
2103 self.inner
2104 .tasks
2105 .lock()
2106 .map_err(|_| "background task registry lock poisoned".to_string())?
2107 .insert(task_id.clone(), task);
2108
2109 Ok(task_id)
2110 }
2111
2112 pub fn write_pty(
2113 &self,
2114 task_id: &str,
2115 session_id: &str,
2116 input: &[u8],
2117 ) -> Result<usize, String> {
2118 let task = self
2119 .task_for_session(task_id, session_id)
2120 .ok_or_else(|| "task_not_found".to_string())?;
2121
2122 let writer = {
2123 let state = task
2124 .state
2125 .lock()
2126 .map_err(|_| "background task lock poisoned".to_string())?;
2127 if state.metadata.mode != BgMode::Pty {
2128 return Err("task_not_pty".to_string());
2129 }
2130 if state.metadata.status.is_terminal() {
2131 return Err("task_exited".to_string());
2132 }
2133 match &state.runtime {
2134 TaskRuntime::Pty(Some(runtime)) => Arc::clone(&runtime.writer),
2135 TaskRuntime::Pty(None) => return Err("task_exited".to_string()),
2136 TaskRuntime::Piped(_) => return Err("task_not_pty".to_string()),
2137 }
2138 };
2139
2140 let mut writer = writer
2141 .lock()
2142 .map_err(|_| "PTY writer lock poisoned".to_string())?;
2143 writer
2144 .write_all(input)
2145 .map_err(|error| format!("failed to write to PTY: {error}"))?;
2146 writer
2147 .flush()
2148 .map_err(|error| format!("failed to flush PTY writer: {error}"))?;
2149 Ok(input.len())
2150 }
2151
2152 pub fn replay_session(&self, storage_dir: &Path, session_id: &str) -> Result<(), String> {
2153 self.replay_session_inner(storage_dir, session_id, None)
2154 }
2155
2156 #[doc(hidden)]
2164 pub fn begin_session_recovery(
2165 &self,
2166 storage_dir: &Path,
2167 session_id: &str,
2168 ) -> SessionRecoveryGuard {
2169 let key = session_recovery_key(storage_dir, session_id);
2170 if let Some(key) = key.as_ref() {
2171 self.inner
2172 .session_recovery
2173 .lock()
2174 .unwrap_or_else(std::sync::PoisonError::into_inner)
2175 .entry(key.clone())
2176 .or_default()
2177 .in_flight += 1;
2178 }
2179 SessionRecoveryGuard {
2180 registry: self.clone(),
2181 key,
2182 }
2183 }
2184
2185 fn session_recovery_state(&self, session_dir: &Path) -> SessionRecovery {
2187 let Some(key) = session_dir.file_name().and_then(|name| name.to_str()) else {
2188 return SessionRecovery::default();
2189 };
2190 self.inner
2191 .session_recovery
2192 .lock()
2193 .unwrap_or_else(std::sync::PoisonError::into_inner)
2194 .get(key)
2195 .copied()
2196 .unwrap_or_default()
2197 }
2198
2199 #[doc(hidden)]
2202 pub fn persisted_gc_thread(&self) -> Option<String> {
2203 self.inner
2204 .persisted_gc_thread
2205 .lock()
2206 .unwrap_or_else(std::sync::PoisonError::into_inner)
2207 .clone()
2208 }
2209
2210 pub fn replay_session_for_project(
2211 &self,
2212 storage_dir: &Path,
2213 session_id: &str,
2214 project_root: &Path,
2215 ) -> Result<(), String> {
2216 self.replay_session_inner(storage_dir, session_id, Some(project_root))
2217 }
2218
2219 fn retire_orphaned_watch_tombstones(&self, binding_session_id: &str) -> Result<(), String> {
2220 let Some((harness, pool)) = self.db_harness_and_pool() else {
2221 return Ok(());
2222 };
2223 let retired = {
2224 let conn = pool
2225 .lock()
2226 .map_err(|_| "background task database lock poisoned".to_string())?;
2227 let rows = crate::db::bash_watches::list_bash_pattern_watches(&conn, &harness)
2228 .map_err(|error| format!("failed to inspect persisted bash watches: {error}"))?;
2229 let mut retired = Vec::new();
2230 for row in rows {
2231 let task_exists = match crate::db::bash_tasks::get_bash_task(
2232 &conn,
2233 &harness,
2234 &row.session_id,
2235 &row.task_id,
2236 ) {
2237 Ok(task) => task.is_some(),
2238 Err(_) => true,
2243 };
2244 let is_erased_target = !task_exists
2245 || (row.pending_match
2246 && row.match_text.as_deref() == Some(WATCH_TARGET_ERASED_TEXT));
2247 if !is_erased_target
2248 || !self.should_retire_foreign_delivery(&row.session_id, binding_session_id)
2249 {
2250 continue;
2251 }
2252 let deleted = crate::db::bash_watches::delete_bash_pattern_watch(
2253 &conn,
2254 &harness,
2255 &row.session_id,
2256 &row.task_id,
2257 &row.watch_id,
2258 )
2259 .map_err(|error| {
2260 format!(
2261 "failed to retire orphaned bash watch {}/{}: {error}",
2262 row.task_id, row.watch_id
2263 )
2264 })?;
2265 if deleted > 0 {
2266 retired.push((row.session_id, row.task_id));
2267 }
2268 }
2269 retired
2270 };
2271
2272 for (originating_session_id, task_id) in retired {
2273 self.clear_task_watch_state(&task_id);
2274 if let Ok(mut registry) = self.inner.watch_registry.lock() {
2275 registry.forget_erased_task(&task_id);
2276 }
2277 crate::slog_info!(
2278 "retired orphaned background watch tombstone: task_id={} originating_session={}",
2279 task_id,
2280 originating_session_id
2281 );
2282 }
2283 Ok(())
2284 }
2285
2286 fn retire_already_reaped_orphaned_completion(
2287 &self,
2288 originating_session_id: &str,
2289 task_id: &str,
2290 ) {
2291 if let Some((harness, pool)) = self.db_harness_and_pool() {
2292 match pool.lock() {
2293 Ok(conn) => {
2294 if let Err(error) = crate::db::bash_tasks::delete_bash_task(
2295 &conn,
2296 &harness,
2297 originating_session_id,
2298 task_id,
2299 ) {
2300 crate::slog_warn!(
2301 "failed to delete already-reaped orphaned background completion row: task_id={task_id} error={error}"
2302 );
2303 }
2304 }
2305 Err(_) => crate::slog_warn!(
2306 "failed to delete already-reaped orphaned background completion row: task_id={task_id} error=database_lock_poisoned"
2307 ),
2308 }
2309 }
2310 let _ = self.remove_pending_completion(task_id);
2311 self.ack_persisted_watches_for_task(originating_session_id, task_id, true);
2312 crate::slog_warn!(
2313 "orphaned completion already reaped: task_id={task_id} reason=layout_missing"
2314 );
2315 }
2316
2317 fn retire_rehydrated_orphaned_completion(
2318 &self,
2319 task: &Arc<BgTask>,
2320 binding_session_id: &str,
2321 ) -> Result<(), String> {
2322 let should_retire = task
2323 .state
2324 .lock()
2325 .map_err(|_| "background task lock poisoned".to_string())
2326 .map(|state| {
2327 state.metadata.status.is_terminal()
2328 && !state.metadata.completion_delivered
2329 && (self.should_retire_foreign_delivery(&task.session_id, binding_session_id)
2330 || !task.paths.dir.exists())
2331 })?;
2332 if !should_retire {
2333 return Ok(());
2334 }
2335
2336 match task.set_completion_delivered(true, self) {
2337 Ok(()) => {
2338 let _ = self.remove_pending_completion(&task.task_id);
2339 self.ack_persisted_watches_for_task(&task.session_id, &task.task_id, true);
2340 crate::slog_info!(
2341 "retired orphaned background completion: task_id={} originating_session={}",
2342 task.task_id,
2343 task.session_id
2344 );
2345 }
2346 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
2347 if let Ok(mut state) = task.state.lock() {
2348 state.metadata.completion_delivered = true;
2349 }
2350 self.retire_already_reaped_orphaned_completion(&task.session_id, &task.task_id);
2351 }
2352 Err(error) => {
2353 return Err(format!("failed to update completion delivery: {error}"));
2354 }
2355 }
2356 Ok(())
2357 }
2358
2359 fn enqueue_replay_completion_if_needed(
2360 &self,
2361 metadata: &mut PersistedTask,
2362 paths: &TaskPaths,
2363 binding_session_id: &str,
2364 ) -> Result<(), String> {
2365 if metadata.status.is_terminal()
2366 && !metadata.completion_delivered
2367 && self.should_retire_foreign_delivery(&metadata.session_id, binding_session_id)
2368 {
2369 metadata.completion_delivered = true;
2370 self.persist_task(paths, metadata).map_err(|error| {
2371 format!(
2372 "failed to retire orphaned background completion {}: {error}",
2373 metadata.task_id
2374 )
2375 })?;
2376 let _ = self.remove_pending_completion(&metadata.task_id);
2377 self.ack_persisted_watches_for_task(&metadata.session_id, &metadata.task_id, true);
2378 crate::slog_info!(
2379 "retired orphaned background completion: task_id={} originating_session={}",
2380 metadata.task_id,
2381 metadata.session_id
2382 );
2383 return Ok(());
2384 }
2385
2386 self.enqueue_completion_if_needed(metadata, Some(paths), false);
2387 Ok(())
2388 }
2389
2390 fn replay_session_inner(
2391 &self,
2392 storage_dir: &Path,
2393 session_id: &str,
2394 project_root: Option<&Path>,
2395 ) -> Result<(), String> {
2396 let mut recovering = vec![self.begin_session_recovery(storage_dir, session_id)];
2401 self.record_live_delivery_session(session_id);
2405 self.retire_orphaned_watch_tombstones(session_id)?;
2406 self.start_watchdog();
2407 if !self.inner.persisted_gc_started.swap(true, Ordering::SeqCst) {
2408 let registry = self.clone();
2420 let storage_dir = storage_dir.to_path_buf();
2421 let spawned = std::thread::Builder::new()
2422 .name("aft-bash-task-gc".to_string())
2423 .spawn(move || {
2424 if let Err(error) = registry.maybe_gc_persisted(&storage_dir) {
2425 crate::slog_warn!("failed to GC persisted background bash tasks: {error}");
2426 }
2427 });
2428 if let Err(error) = spawned {
2429 crate::slog_warn!("failed to spawn persisted background task GC: {error}");
2430 }
2431 }
2432
2433 let canonical_project = project_root.map(canonicalized_path);
2434 let tasks = match self.replay_session_from_db(session_id, project_root) {
2446 Some(Ok(tasks)) if !tasks.is_empty() => tasks,
2447 Some(Ok(_)) => {
2448 let disk_tasks = self.replay_session_from_disk(storage_dir, session_id)?;
2449 if !disk_tasks.is_empty() {
2450 crate::slog_info!(
2451 "bash task replay: 0 in DB for session {}, {} from disk fallback",
2452 session_id,
2453 disk_tasks.len()
2454 );
2455 }
2456 disk_tasks
2457 }
2458 Some(Err(error)) => {
2459 crate::slog_warn!(
2460 "bash task replay DB lookup failed for session {}; falling back to disk: {}",
2461 session_id,
2462 error
2463 );
2464 self.replay_session_from_disk(storage_dir, session_id)?
2465 }
2466 None => {
2467 self.replay_session_from_disk(storage_dir, session_id)?
2469 }
2470 };
2471
2472 let mut guarded = HashSet::from([session_id.to_string()]);
2476 for metadata in &tasks {
2477 if guarded.insert(metadata.session_id.clone()) {
2478 recovering.push(self.begin_session_recovery(storage_dir, &metadata.session_id));
2479 }
2480 }
2481
2482 for mut metadata in tasks {
2483 if project_root.is_none() && metadata.session_id != session_id {
2484 continue;
2485 }
2486 if let Some(canonical_project) = canonical_project.as_deref() {
2487 let metadata_project = metadata.project_root.as_deref().map(canonicalized_path);
2488 if metadata_project.as_deref() != Some(canonical_project) {
2489 continue;
2490 }
2491 }
2492
2493 if validate_task_id(&metadata.task_id).is_err() {
2494 crate::slog_warn!(
2495 "ignoring persisted background task with invalid id {:?}",
2496 metadata.task_id
2497 );
2498 continue;
2499 }
2500 if let Some(task) = self.task(&metadata.task_id) {
2504 self.retire_rehydrated_orphaned_completion(&task, session_id)?;
2505 continue;
2506 }
2507 let session_dir = session_tasks_dir(storage_dir, &metadata.session_id);
2508 let resolved = match resolve_task_layout(&session_dir, &metadata.task_id) {
2509 Ok(task) => task,
2510 Err(error) => {
2511 if error.kind() == std::io::ErrorKind::NotFound
2512 && !Self::persisted_task_process_is_alive(&metadata)
2513 && (self.should_retire_foreign_delivery(&metadata.session_id, session_id)
2514 || &metadata.session_id != session_id)
2515 {
2516 self.retire_already_reaped_orphaned_completion(
2517 &metadata.session_id,
2518 &metadata.task_id,
2519 );
2520 continue;
2521 }
2522 if Self::persisted_task_process_is_alive(&metadata) {
2523 crate::slog_warn!(
2524 "refusing to quarantine unresolved live background task {}: {error}",
2525 metadata.task_id
2526 );
2527 continue;
2528 }
2529 crate::slog_warn!(
2530 "quarantining unresolved background task {}: {error}",
2531 metadata.task_id
2532 );
2533 let _ = quarantine_task_layout(
2534 storage_dir,
2535 &session_dir,
2536 &metadata.task_id,
2537 "invalid",
2538 );
2539 continue;
2540 }
2541 };
2542 match read_task_at(&resolved) {
2543 Ok(disk)
2544 if disk.task_id == metadata.task_id
2545 && disk.session_id == metadata.session_id => {}
2546 Ok(_) | Err(_) => {
2547 if Self::persisted_task_process_is_alive(&metadata) {
2548 crate::slog_warn!(
2549 "refusing to quarantine mismatched live background task {}",
2550 metadata.task_id
2551 );
2552 continue;
2553 }
2554 let _ = quarantine_task_layout(
2555 storage_dir,
2556 &session_dir,
2557 &metadata.task_id,
2558 "mismatch",
2559 );
2560 continue;
2561 }
2562 }
2563 let paths = resolved.paths;
2564 match metadata.status {
2565 BgTaskStatus::Starting => {
2566 let completion_was_delivered = metadata.completion_delivered;
2567 metadata.mark_terminal(
2568 BgTaskStatus::Failed,
2569 None,
2570 Some("spawn aborted".to_string()),
2571 );
2572 metadata.completion_delivered |= completion_was_delivered;
2573 let _ = self.persist_task(&paths, &metadata);
2574 self.enqueue_replay_completion_if_needed(&mut metadata, &paths, session_id)?;
2575 self.insert_rehydrated_task(metadata, paths, true)?;
2576 }
2577 BgTaskStatus::Running | BgTaskStatus::Killing => {
2578 if metadata.mode == BgMode::Pty {
2579 if let Ok(Some(marker)) = read_exit_marker(&paths) {
2580 let completion_was_delivered = metadata.completion_delivered;
2581 metadata = terminal_metadata_from_marker(metadata, marker, None);
2582 metadata.completion_delivered |= completion_was_delivered;
2583 let _ = self.persist_task(&paths, &metadata);
2584 self.enqueue_replay_completion_if_needed(
2585 &mut metadata,
2586 &paths,
2587 session_id,
2588 )?;
2589 self.insert_rehydrated_task(metadata, paths, true)?;
2590 } else if metadata.status.is_terminal() {
2591 self.insert_rehydrated_task(metadata, paths, true)?;
2592 } else {
2593 let completion_was_delivered = metadata.completion_delivered;
2594 metadata.mark_terminal(
2595 BgTaskStatus::Killed,
2596 None,
2597 Some("pty_lost_on_bridge_restart".to_string()),
2598 );
2599 metadata.completion_delivered |= completion_was_delivered;
2600 let _ = self.persist_task(&paths, &metadata);
2601 self.enqueue_replay_completion_if_needed(
2602 &mut metadata,
2603 &paths,
2604 session_id,
2605 )?;
2606 self.insert_rehydrated_task(metadata, paths, true)?;
2607 }
2608 } else if let Ok(Some(marker)) = read_exit_marker(&paths) {
2609 let reason = (metadata.status == BgTaskStatus::Killing).then(|| {
2610 "recovered from inconsistent killing state on replay".to_string()
2611 });
2612 if reason.is_some() {
2613 crate::slog_warn!("background task {} had killing state with exit marker; preferring marker",
2614 metadata.task_id);
2615 }
2616 let completion_was_delivered = metadata.completion_delivered;
2617 metadata = terminal_metadata_from_marker(metadata, marker, reason);
2618 metadata.completion_delivered |= completion_was_delivered;
2619 let _ = self.persist_task(&paths, &metadata);
2620 self.enqueue_replay_completion_if_needed(
2621 &mut metadata,
2622 &paths,
2623 session_id,
2624 )?;
2625 self.insert_rehydrated_task(metadata, paths, true)?;
2626 } else if metadata.status == BgTaskStatus::Killing {
2627 let _ = write_kill_marker_if_absent(&paths);
2628 let completion_was_delivered = metadata.completion_delivered;
2629 metadata.mark_terminal(
2630 BgTaskStatus::Killed,
2631 None,
2632 Some("recovered from inconsistent killing state on replay".to_string()),
2633 );
2634 metadata.completion_delivered |= completion_was_delivered;
2635 let _ = self.persist_task(&paths, &metadata);
2636 self.enqueue_replay_completion_if_needed(
2637 &mut metadata,
2638 &paths,
2639 session_id,
2640 )?;
2641 self.insert_rehydrated_task(metadata, paths, true)?;
2642 } else if Self::persisted_task_process_is_alive(&metadata) {
2643 self.insert_rehydrated_task(metadata, paths, true)?;
2644 } else {
2645 let completion_was_delivered = metadata.completion_delivered;
2646 metadata.mark_terminal(
2647 BgTaskStatus::FateUnknown,
2648 None,
2649 Some(restart_fate_unknown_reason(&metadata, &paths)),
2650 );
2651 metadata.completion_delivered |= completion_was_delivered;
2652 let _ = self.persist_task(&paths, &metadata);
2653 self.enqueue_replay_completion_if_needed(
2654 &mut metadata,
2655 &paths,
2656 session_id,
2657 )?;
2658 self.insert_rehydrated_task(metadata, paths, true)?;
2659 }
2660 }
2661 _ if metadata.status.is_terminal() => {
2662 self.enqueue_replay_completion_if_needed(&mut metadata, &paths, session_id)?;
2668 self.insert_rehydrated_task(metadata, paths, true)?;
2669 }
2670 _ => {}
2671 }
2672 }
2673
2674 drop(recovering);
2677 Ok(())
2678 }
2679
2680 fn replay_session_from_db(
2681 &self,
2682 session_id: &str,
2683 project_root: Option<&Path>,
2684 ) -> Option<Result<Vec<PersistedTask>, String>> {
2685 let pool = self
2686 .inner
2687 .db_pool
2688 .read()
2689 .ok()
2690 .and_then(|slot| slot.clone())?;
2691 let harness = self
2692 .inner
2693 .db_harness
2694 .read()
2695 .ok()
2696 .and_then(|slot| slot.clone())?;
2697 let conn = match pool.lock() {
2698 Ok(conn) => conn,
2699 Err(_) => return Some(Err("db mutex poisoned".to_string())),
2700 };
2701 let rows = if let Some(project_root) = project_root {
2702 let project_key = crate::path_identity::project_scope_key(project_root);
2703 crate::db::bash_tasks::list_replayable_bash_tasks_for_project(
2704 &conn,
2705 &harness,
2706 &project_key,
2707 )
2708 } else {
2709 crate::db::bash_tasks::list_bash_tasks_for_session(&conn, &harness, session_id)
2710 };
2711 Some(
2712 rows.map(|rows| rows.into_iter().map(PersistedTask::from).collect())
2713 .map_err(|error| error.to_string()),
2714 )
2715 }
2716
2717 fn replay_session_from_disk(
2718 &self,
2719 storage_dir: &Path,
2720 session_id: &str,
2721 ) -> Result<Vec<PersistedTask>, String> {
2722 let dir = session_tasks_dir(storage_dir, session_id);
2723 if !dir.exists() {
2724 return Ok(Vec::new());
2725 }
2726
2727 let (task_ids, invalid_entries) = discover_task_ids(&dir)
2728 .map_err(|error| format!("failed to discover background task layouts: {error}"))?;
2729 for entry in invalid_entries {
2730 if let Err(error) = quarantine_invalid_entry(storage_dir, &dir, &entry) {
2731 crate::slog_warn!(
2732 "failed to quarantine invalid background task entry {:?}: {error}",
2733 entry
2734 );
2735 }
2736 }
2737
2738 let mut tasks = Vec::new();
2739 for task_id in task_ids {
2740 let task = match resolve_task_layout(&dir, &task_id) {
2741 Ok(task) => task,
2742 Err(error)
2743 if error.kind() == std::io::ErrorKind::NotFound
2744 && uninitialized_layout_is_recent(
2745 &dir,
2746 &task_id,
2747 Duration::from_secs(5 * 60),
2748 )
2749 .unwrap_or(false) =>
2750 {
2751 continue;
2752 }
2753 Err(error) => {
2754 if self.db_has_live_process_for_task(&task_id) {
2755 crate::slog_warn!(
2756 "refusing to quarantine unresolved live background task {task_id} during replay: {error}"
2757 );
2758 continue;
2759 }
2760 crate::slog_warn!(
2761 "quarantining unresolved background task {task_id} during replay: {error}"
2762 );
2763 let _ = quarantine_task_layout(storage_dir, &dir, &task_id, "invalid");
2764 continue;
2765 }
2766 };
2767 match read_task_at(&task) {
2768 Ok(metadata) if metadata.session_id == session_id => tasks.push(metadata),
2769 Ok(_) => {
2770 crate::slog_warn!(
2771 "quarantining background task {task_id} with mismatched session metadata"
2772 );
2773 let _ = quarantine_task_layout(storage_dir, &dir, &task_id, "mismatch");
2774 }
2775 Err(error) => {
2776 if self.db_has_live_process_for_task(&task_id) {
2777 crate::slog_warn!(
2778 "refusing to quarantine unreadable live background task {task_id} during replay: {error}"
2779 );
2780 continue;
2781 }
2782 crate::slog_warn!(
2783 "quarantining invalid background task metadata {task_id} during replay: {error}"
2784 );
2785 let _ = quarantine_task_layout(storage_dir, &dir, &task_id, "invalid");
2786 }
2787 }
2788 }
2789 Ok(tasks)
2790 }
2791
2792 pub fn register_watch(
2793 &self,
2794 task_id: String,
2795 pattern: WatchPattern,
2796 once: bool,
2797 ) -> Result<String, &'static str> {
2798 let task = self.task(&task_id).ok_or("task_not_found")?;
2799 self.record_live_delivery_session(&task.session_id);
2800 validate_task_id(&task_id).map_err(|_| "invalid_task_id")?;
2801 let (mode, terminal_at_registration) = task
2802 .state
2803 .lock()
2804 .map(|state| {
2805 (
2806 state.metadata.mode.clone(),
2807 state.metadata.status.is_terminal(),
2808 )
2809 })
2810 .map_err(|_| "background_task_lock_poisoned")?;
2811 let mut stdout = (mode == BgMode::Pipes)
2812 .then(|| open_task_artifact(&task.paths, TaskArtifact::Stdout))
2813 .transpose()
2814 .map_err(|_| "artifact_refused")?;
2815 let mut stderr = (mode == BgMode::Pipes)
2816 .then(|| open_task_artifact(&task.paths, TaskArtifact::Stderr))
2817 .transpose()
2818 .map_err(|_| "artifact_refused")?;
2819 let mut pty = (mode == BgMode::Pty)
2820 .then(|| open_task_artifact(&task.paths, TaskArtifact::Pty))
2821 .transpose()
2822 .map_err(|_| "artifact_refused")?;
2823
2824 let mut terminal_matches = Vec::new();
2825 let scanned_terminal = terminal_at_registration;
2826 let watch_id = {
2827 let mut registry = self
2828 .inner
2829 .watch_registry
2830 .lock()
2831 .map_err(|_| "watch_registry_poisoned")?;
2832 let watch_id = registry.register(task_id.clone(), pattern.clone(), once)?;
2833 match &mode {
2834 BgMode::Pipes => {
2835 let stdout_key = format!("{task_id}:stdout");
2836 let stderr_key = format!("{task_id}:stderr");
2837 if terminal_at_registration {
2838 registry.set_file_cursor(&stdout_key, 0);
2839 registry.set_file_cursor(&stderr_key, 0);
2840 terminal_matches.extend(registry.scan_file_new_bytes(
2841 &stdout_key,
2842 &task_id,
2843 stdout.as_mut().expect("pipe stdout opened"),
2844 ));
2845 terminal_matches.extend(registry.scan_file_new_bytes(
2846 &stderr_key,
2847 &task_id,
2848 stderr.as_mut().expect("pipe stderr opened"),
2849 ));
2850 } else {
2851 registry.prime_file_cursor(
2852 &stdout_key,
2853 stdout.as_ref().expect("pipe stdout opened"),
2854 );
2855 registry.prime_file_cursor(
2856 &stderr_key,
2857 stderr.as_ref().expect("pipe stderr opened"),
2858 );
2859 }
2860 }
2861 BgMode::Pty => {
2862 let pty_key = format!("{task_id}:pty");
2863 if terminal_at_registration {
2864 registry.set_file_cursor(&pty_key, 0);
2865 terminal_matches.extend(registry.scan_file_new_bytes(
2866 &pty_key,
2867 &task_id,
2868 pty.as_mut().expect("PTY artifact opened"),
2869 ));
2870 } else {
2871 registry.prime_file_cursor(
2872 &pty_key,
2873 pty.as_ref().expect("PTY artifact opened"),
2874 );
2875 }
2876 }
2877 }
2878 watch_id
2879 };
2880
2881 let (stdout_offset, stderr_offset, pty_offset) = self.watch_stream_cursors(&task_id);
2882 self.persist_watch_registration(
2883 &task.session_id,
2884 &task_id,
2885 &watch_id,
2886 &pattern,
2887 once,
2888 stdout_offset,
2889 stderr_offset,
2890 pty_offset,
2891 );
2892
2893 if task.is_terminal() {
2894 if !scanned_terminal {
2895 terminal_matches = {
2896 let mut registry = self
2897 .inner
2898 .watch_registry
2899 .lock()
2900 .map_err(|_| "watch_registry_poisoned")?;
2901 match &mode {
2902 BgMode::Pipes => {
2903 let stdout_key = format!("{task_id}:stdout");
2904 let stderr_key = format!("{task_id}:stderr");
2905 registry.set_file_cursor(&stdout_key, 0);
2906 registry.set_file_cursor(&stderr_key, 0);
2907 let mut matches = registry.scan_file_new_bytes(
2908 &stdout_key,
2909 &task_id,
2910 stdout.as_mut().expect("pipe stdout opened"),
2911 );
2912 matches.extend(registry.scan_file_new_bytes(
2913 &stderr_key,
2914 &task_id,
2915 stderr.as_mut().expect("pipe stderr opened"),
2916 ));
2917 matches
2918 }
2919 BgMode::Pty => {
2920 let pty_key = format!("{task_id}:pty");
2921 registry.set_file_cursor(&pty_key, 0);
2922 registry.scan_file_new_bytes(
2923 &pty_key,
2924 &task_id,
2925 pty.as_mut().expect("PTY artifact opened"),
2926 )
2927 }
2928 }
2929 };
2930 }
2931
2932 let (stdout_offset, stderr_offset, pty_offset) = self.watch_stream_cursors(&task_id);
2933 let (watch_controlled, watch_matched) = self.task_watch_state(&task_id);
2934 if terminal_matches.is_empty() && (!watch_controlled || watch_matched) {
2935 if watch_matched {
2936 self.clear_task_watch_state(&task_id);
2937 }
2938 return Ok(watch_id);
2939 }
2940
2941 let completion = self
2942 .remove_pending_completion(&task_id)
2943 .or_else(|| self.completion_snapshot_for_task(&task));
2944 if terminal_matches.is_empty() {
2945 if let Some(completion) = completion.as_ref() {
2946 self.record_bash_watch_exit(completion, true);
2947 }
2948 } else {
2949 for pattern_match in &terminal_matches {
2950 self.persist_watch_match(
2951 &task.session_id,
2952 &task_id,
2953 pattern_match,
2954 stdout_offset,
2955 stderr_offset,
2956 pty_offset,
2957 );
2958 self.emit_bash_pattern_match(&task.session_id, pattern_match.clone());
2959 }
2960 }
2961 self.clear_task_watch_state(&task_id);
2962 }
2963
2964 Ok(watch_id)
2965 }
2966
2967 pub fn unregister_watch(&self, task_id: &str, watch_id: &str) {
2968 let session_id = self.task(task_id).map(|task| task.session_id.clone());
2969 if let Ok(mut registry) = self.inner.watch_registry.lock() {
2970 registry.unregister(task_id, watch_id);
2971 }
2972 if let Some(session_id) = session_id {
2973 self.delete_persisted_watch(&session_id, task_id, watch_id);
2974 }
2975 }
2976
2977 pub fn active_watch_count(&self, task_id: &str) -> usize {
2978 self.inner
2979 .watch_registry
2980 .lock()
2981 .map(|registry| registry.active_count(task_id))
2982 .unwrap_or(0)
2983 }
2984
2985 fn task_watch_state(&self, task_id: &str) -> (bool, bool) {
2986 self.inner
2987 .watch_registry
2988 .lock()
2989 .map(|registry| {
2990 (
2991 registry.has_controlled_task(task_id),
2992 registry.has_matched_task(task_id),
2993 )
2994 })
2995 .unwrap_or((false, false))
2996 }
2997
2998 fn task_has_watch_control(&self, task_id: &str) -> bool {
2999 self.inner
3000 .watch_registry
3001 .lock()
3002 .map(|registry| registry.has_controlled_task(task_id))
3003 .unwrap_or(false)
3004 }
3005
3006 fn clear_task_watch_state(&self, task_id: &str) {
3007 if let Ok(mut registry) = self.inner.watch_registry.lock() {
3008 registry.clear_task(task_id);
3009 }
3010 }
3011
3012 pub(crate) fn scan_task_watch_output(&self, task: &Arc<BgTask>) {
3013 let mode = match task.state.lock() {
3014 Ok(state) => state.metadata.mode.clone(),
3015 Err(_) => return,
3016 };
3017 let mut stdout = (mode == BgMode::Pipes)
3018 .then(|| open_task_artifact(&task.paths, TaskArtifact::Stdout))
3019 .transpose()
3020 .ok()
3021 .flatten();
3022 let mut stderr = (mode == BgMode::Pipes)
3023 .then(|| open_task_artifact(&task.paths, TaskArtifact::Stderr))
3024 .transpose()
3025 .ok()
3026 .flatten();
3027 let mut pty = (mode == BgMode::Pty)
3028 .then(|| open_task_artifact(&task.paths, TaskArtifact::Pty))
3029 .transpose()
3030 .ok()
3031 .flatten();
3032 let mut matches = Vec::new();
3033 if let Ok(mut registry) = self.inner.watch_registry.lock() {
3034 match mode {
3035 BgMode::Pipes => {
3036 let (Some(stdout), Some(stderr)) = (stdout.as_mut(), stderr.as_mut()) else {
3037 return;
3038 };
3039 let stdout_key = format!("{}:stdout", task.task_id);
3040 let stderr_key = format!("{}:stderr", task.task_id);
3041 matches.extend(registry.scan_file_new_bytes(
3042 &stdout_key,
3043 &task.task_id,
3044 stdout,
3045 ));
3046 matches.extend(registry.scan_file_new_bytes(
3047 &stderr_key,
3048 &task.task_id,
3049 stderr,
3050 ));
3051 }
3052 BgMode::Pty => {
3053 let Some(pty) = pty.as_mut() else {
3054 return;
3055 };
3056 let pty_key = format!("{}:pty", task.task_id);
3057 matches.extend(registry.scan_file_new_bytes(&pty_key, &task.task_id, pty));
3058 }
3059 }
3060 }
3061 let (stdout_offset, stderr_offset, pty_offset) = self.watch_stream_cursors(&task.task_id);
3062 if matches.is_empty() {
3063 if self.task_has_watch_control(&task.task_id) {
3066 self.persist_task_watch_cursors(
3067 &task.session_id,
3068 &task.task_id,
3069 stdout_offset,
3070 stderr_offset,
3071 pty_offset,
3072 );
3073 }
3074 return;
3075 }
3076 for pattern_match in matches {
3077 self.persist_watch_match(
3078 &task.session_id,
3079 &task.task_id,
3080 &pattern_match,
3081 stdout_offset,
3082 stderr_offset,
3083 pty_offset,
3084 );
3085 self.emit_bash_pattern_match(&task.session_id, pattern_match);
3086 }
3087 self.persist_task_watch_cursors(
3088 &task.session_id,
3089 &task.task_id,
3090 stdout_offset,
3091 stderr_offset,
3092 pty_offset,
3093 );
3094 }
3095
3096 pub(crate) fn observed_status(
3097 &self,
3098 task_id: &str,
3099 session_id: &str,
3100 preview_bytes: usize,
3101 ) -> Option<BgTaskSnapshot> {
3102 validate_task_id(task_id).ok()?;
3103 let task = self.task_for_session(task_id, session_id)?;
3104 Some(self.snapshot_with_terminal_cache(&task, preview_bytes))
3105 }
3106
3107 pub fn status(
3108 &self,
3109 task_id: &str,
3110 session_id: &str,
3111 project_root: Option<&Path>,
3112 storage_dir: Option<&Path>,
3113 preview_bytes: usize,
3114 ) -> Option<BgTaskSnapshot> {
3115 validate_task_id(task_id).ok()?;
3116 let terminal_db_fallback_allowed = storage_dir
3117 .is_some_and(|storage_dir| task_bundle_is_absent(storage_dir, session_id, task_id));
3118 let mut task = self.task_for_session(task_id, session_id);
3119 if task.is_none() {
3120 if let Some(storage_dir) = storage_dir {
3121 let _ = if let Some(project_root) = project_root {
3122 self.replay_session_for_project(storage_dir, session_id, project_root)
3123 } else {
3124 self.replay_session(storage_dir, session_id)
3125 };
3126 task = self.task_for_session(task_id, session_id);
3127 }
3128 }
3129 let Some(task) = task else {
3130 if terminal_db_fallback_allowed {
3131 if let Some(snapshot) = storage_dir.and_then(|storage_dir| {
3132 self.terminal_db_status_for_session(session_id, task_id, storage_dir)
3133 }) {
3134 return Some(snapshot);
3135 }
3136 }
3137 return self.status_relaxed(
3138 task_id,
3139 session_id,
3140 project_root?,
3141 storage_dir?,
3142 preview_bytes,
3143 terminal_db_fallback_allowed,
3144 );
3145 };
3146 let _ = self.poll_task(&task);
3147 Some(self.snapshot_with_terminal_cache(&task, preview_bytes))
3148 }
3149
3150 fn status_relaxed_task(
3151 &self,
3152 task_id: &str,
3153 project_root: &Path,
3154 storage_dir: &Path,
3155 ) -> Option<Arc<BgTask>> {
3156 validate_task_id(task_id).ok()?;
3157 let canonical_project = canonicalized_path(project_root);
3158 match self.lookup_relaxed_task_from_db(task_id, project_root) {
3159 Some(Ok(Some(row))) => {
3160 let metadata = PersistedTask::from(row);
3161 if let Some(task) = self.task(task_id) {
3162 let matches_project = task
3163 .state
3164 .lock()
3165 .map(|state| {
3166 state
3167 .metadata
3168 .project_root
3169 .as_deref()
3170 .map(canonicalized_path)
3171 .as_deref()
3172 == Some(canonical_project.as_path())
3173 })
3174 .unwrap_or(false);
3175 return matches_project.then_some(task);
3176 }
3177 let resolved = resolve_task_layout(
3178 &session_tasks_dir(storage_dir, &metadata.session_id),
3179 &metadata.task_id,
3180 )
3181 .ok()?;
3182 let disk = read_task_at(&resolved).ok()?;
3183 if disk.task_id != metadata.task_id || disk.session_id != metadata.session_id {
3184 return None;
3185 }
3186 if self
3187 .insert_rehydrated_task(metadata, resolved.paths, true)
3188 .is_err()
3189 {
3190 return None;
3191 }
3192 return self.task(task_id);
3193 }
3194 Some(Ok(None)) => {
3195 crate::slog_info!(
3196 "bash task relaxed DB miss for {}; falling back to disk",
3197 task_id
3198 );
3199 }
3200 Some(Err(error)) => {
3201 crate::slog_warn!(
3202 "bash task relaxed DB lookup failed for {}; falling back to disk: {}",
3203 task_id,
3204 error
3205 );
3206 }
3207 None => {
3208 crate::slog_info!(
3209 "bash task relaxed DB unavailable for {}; falling back to disk",
3210 task_id
3211 );
3212 }
3213 }
3214 let root = storage_dir.join("bash-tasks");
3215 let entries = fs::read_dir(&root).ok()?;
3216 for entry in entries.flatten() {
3217 let dir = entry.path();
3218 if !dir.is_dir() {
3219 continue;
3220 }
3221 let resolved = match resolve_task_layout(&dir, task_id) {
3222 Ok(task) => task,
3223 Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue,
3224 Err(error) => {
3225 if self.db_has_live_process_for_task(task_id) {
3226 crate::slog_warn!(
3227 "refusing to quarantine unresolved live background task {task_id} during relaxed lookup: {error}"
3228 );
3229 continue;
3230 }
3231 crate::slog_warn!(
3232 "quarantining unresolved background task {task_id} during relaxed lookup: {error}"
3233 );
3234 let _ = quarantine_task_layout(storage_dir, &dir, task_id, "invalid");
3235 continue;
3236 }
3237 };
3238 let metadata = match read_task_at(&resolved) {
3239 Ok(metadata) => metadata,
3240 Err(error) => {
3241 if self.db_has_live_process_for_task(task_id) {
3242 crate::slog_warn!(
3243 "refusing to quarantine unreadable live background task {task_id} during relaxed lookup: {error}"
3244 );
3245 continue;
3246 }
3247 crate::slog_warn!(
3248 "quarantining invalid background task metadata {task_id} during relaxed lookup: {error}"
3249 );
3250 let _ = quarantine_task_layout(storage_dir, &dir, task_id, "invalid");
3251 continue;
3252 }
3253 };
3254 let metadata_project = metadata.project_root.as_deref().map(canonicalized_path);
3255 if metadata_project.as_deref() != Some(canonical_project.as_path()) {
3256 continue;
3257 }
3258 if let Some(task) = self.task(task_id) {
3259 let matches_project = task
3260 .state
3261 .lock()
3262 .map(|state| {
3263 state
3264 .metadata
3265 .project_root
3266 .as_deref()
3267 .map(canonicalized_path)
3268 .as_deref()
3269 == Some(canonical_project.as_path())
3270 })
3271 .unwrap_or(false);
3272 return matches_project.then_some(task);
3273 }
3274 if self
3275 .insert_rehydrated_task(metadata, resolved.paths, true)
3276 .is_err()
3277 {
3278 return None;
3279 }
3280 return self.task(task_id);
3281 }
3282 None
3283 }
3284
3285 fn lookup_relaxed_task_from_db(
3286 &self,
3287 task_id: &str,
3288 project_root: &Path,
3289 ) -> Option<Result<Option<BashTaskRow>, String>> {
3290 let pool = self
3291 .inner
3292 .db_pool
3293 .read()
3294 .ok()
3295 .and_then(|slot| slot.clone())?;
3296 let harness = self
3297 .inner
3298 .db_harness
3299 .read()
3300 .ok()
3301 .and_then(|slot| slot.clone())?;
3302 let conn = match pool.lock() {
3303 Ok(conn) => conn,
3304 Err(_) => return Some(Err("db mutex poisoned".to_string())),
3305 };
3306 let project_key = crate::path_identity::project_scope_key(project_root);
3307 Some(
3308 crate::db::bash_tasks::find_bash_task_for_project(
3309 &conn,
3310 &harness,
3311 &project_key,
3312 task_id,
3313 )
3314 .map_err(|error| error.to_string()),
3315 )
3316 }
3317
3318 pub(super) fn status_relaxed(
3319 &self,
3320 task_id: &str,
3321 _session_id: &str,
3322 project_root: &Path,
3323 storage_dir: &Path,
3324 preview_bytes: usize,
3325 allow_terminal_db_fallback: bool,
3326 ) -> Option<BgTaskSnapshot> {
3327 let fallback_row = if allow_terminal_db_fallback {
3328 self.lookup_relaxed_task_from_db(task_id, project_root)
3329 } else {
3330 None
3331 }
3332 .and_then(Result::ok)
3333 .flatten()
3334 .filter(|row| task_bundle_is_absent(storage_dir, &row.session_id, &row.task_id));
3335 if let Some(task) = self.status_relaxed_task(task_id, project_root, storage_dir) {
3336 let _ = self.poll_task(&task);
3337 return Some(self.snapshot_with_terminal_cache(&task, preview_bytes));
3338 }
3339 let row = fallback_row?;
3340 let metadata = PersistedTask::from(row.clone());
3341 metadata
3342 .status
3343 .is_terminal()
3344 .then(|| terminal_db_row_snapshot(row, metadata))
3345 }
3346
3347 pub fn kill_relaxed(
3348 &self,
3349 task_id: &str,
3350 project_root: &Path,
3351 storage_dir: &Path,
3352 ) -> Result<BgTaskSnapshot, String> {
3353 let task = self
3354 .status_relaxed_task(task_id, project_root, storage_dir)
3355 .ok_or_else(|| format!("background task not found: {task_id}"))?;
3356 self.kill_with_status(task_id, &task.session_id, BgTaskStatus::Killed)
3357 }
3358
3359 pub fn maybe_gc_persisted(&self, storage_dir: &Path) -> Result<usize, String> {
3360 #[cfg(test)]
3361 self.inner.persisted_gc_runs.fetch_add(1, Ordering::SeqCst);
3362 struct RecordThreadOnExit<'a>(&'a RegistryInner);
3367 impl Drop for RecordThreadOnExit<'_> {
3368 fn drop(&mut self) {
3369 *self
3370 .0
3371 .persisted_gc_thread
3372 .lock()
3373 .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(
3374 std::thread::current()
3375 .name()
3376 .unwrap_or("<unnamed>")
3377 .to_string(),
3378 );
3379 }
3380 }
3381 let _record_thread = RecordThreadOnExit(&self.inner);
3382
3383 let mut deleted = 0usize;
3384
3385 let root = storage_dir.join("bash-tasks");
3386 if root.exists() {
3387 let session_dirs = fs::read_dir(&root).map_err(|e| {
3388 format!(
3389 "failed to read background task root {}: {e}",
3390 root.display()
3391 )
3392 })?;
3393 for session_entry in session_dirs.flatten() {
3394 let session_dir = session_entry.path();
3395 if !session_dir.is_dir() {
3396 continue;
3397 }
3398 let recovery_before = self.session_recovery_state(&session_dir);
3405 if recovery_before.in_flight > 0 {
3406 log::debug!(
3407 "skipping background task session {} during GC: replay recovery in flight",
3408 session_dir.display()
3409 );
3410 continue;
3411 }
3412 let undisturbed = |registry: &Self| {
3417 registry.session_recovery_state(&session_dir) == recovery_before
3418 };
3419 let (task_ids, invalid_entries) = match discover_task_ids(&session_dir) {
3420 Ok(discovery) => discovery,
3421 Err(error) => {
3422 crate::slog_warn!(
3423 "failed to discover background task session {}: {error}",
3424 session_dir.display()
3425 );
3426 continue;
3427 }
3428 };
3429 for entry in invalid_entries {
3430 if !undisturbed(self) {
3431 break;
3432 }
3433 let _ = quarantine_invalid_entry(storage_dir, &session_dir, &entry);
3434 }
3435 for task_id in task_ids {
3436 let resolved = match resolve_task_layout(&session_dir, &task_id) {
3437 Ok(task) => task,
3438 Err(error)
3443 if error.kind() == std::io::ErrorKind::NotFound
3444 && uninitialized_layout_is_recent(
3445 &session_dir,
3446 &task_id,
3447 Duration::from_secs(5 * 60),
3448 )
3449 .unwrap_or(true) =>
3450 {
3451 continue;
3452 }
3453 Err(error) => {
3454 if concurrently_replaced(&error) {
3459 continue;
3460 }
3461 if self.db_has_live_process_for_task(&task_id) {
3462 crate::slog_warn!(
3463 "refusing to quarantine unresolved live background task {task_id} during GC: {error}"
3464 );
3465 continue;
3466 }
3467 if !undisturbed(self) {
3468 continue;
3469 }
3470 crate::slog_warn!(
3471 "quarantining unresolved background task {task_id}: {error}"
3472 );
3473 quarantine_task_layout(storage_dir, &session_dir, &task_id, "invalid")
3474 .map_err(|error| error.to_string())?;
3475 continue;
3476 }
3477 };
3478 if modified_within(&resolved.paths.json, PERSISTED_GC_GRACE) {
3479 continue;
3480 }
3481 let metadata = match read_task_at(&resolved) {
3482 Ok(metadata) => metadata,
3483 Err(error) => {
3484 if concurrently_replaced(&error) {
3485 continue;
3486 }
3487 if self.db_has_live_process_for_task(&task_id) {
3488 crate::slog_warn!(
3489 "refusing to quarantine unreadable live background task {task_id} during GC: {error}"
3490 );
3491 continue;
3492 }
3493 if !undisturbed(self) {
3494 continue;
3495 }
3496 crate::slog_warn!(
3497 "quarantining corrupt background task metadata {task_id}: {error}"
3498 );
3499 quarantine_task_layout(storage_dir, &session_dir, &task_id, "corrupt")
3500 .map_err(|error| error.to_string())?;
3501 continue;
3502 }
3503 };
3504 if !(metadata.status.is_terminal() && metadata.completion_delivered) {
3505 continue;
3506 }
3507 if Self::persisted_task_process_is_alive(&metadata)
3508 || self.db_has_live_process_for_task(&task_id)
3509 {
3510 crate::slog_warn!(
3511 "refusing to delete terminal background task bundle {task_id}: recorded process is still alive"
3512 );
3513 continue;
3514 }
3515 if !undisturbed(self) {
3516 continue;
3517 }
3518 match delete_task_bundle(&resolved.paths) {
3519 Ok(()) => {
3520 self.delete_gc_task_from_db(&metadata);
3521 self.evaluate_erased_watch_targets();
3522 deleted += 1;
3523 log::debug!(
3524 "deleted persisted background task bundle {}",
3525 metadata.task_id
3526 );
3527 }
3528 Err(error) => {
3529 crate::slog_warn!(
3530 "failed to delete background task bundle {}: {error}",
3531 metadata.task_id
3532 );
3533 }
3534 }
3535 }
3536 }
3537 }
3538 gc_quarantine(storage_dir);
3539 Ok(deleted)
3540 }
3541
3542 pub fn list(&self, preview_bytes: usize) -> Vec<BgTaskSnapshot> {
3543 let tasks = self
3544 .inner
3545 .tasks
3546 .lock()
3547 .map(|tasks| tasks.values().cloned().collect::<Vec<_>>())
3548 .unwrap_or_default();
3549 tasks
3550 .into_iter()
3551 .map(|task| {
3552 let _ = self.poll_task(&task);
3553 self.snapshot_with_terminal_cache(&task, preview_bytes)
3554 })
3555 .collect()
3556 }
3557
3558 fn maybe_compress_snapshot(&self, task: &Arc<BgTask>, snapshot: &mut BgTaskSnapshot) {
3564 if !snapshot.info.status.is_terminal() || snapshot.info.mode == BgMode::Pty {
3565 return;
3566 }
3567 if let Some(cache) = self.ensure_terminal_output_cache(task) {
3568 let mut output_preview = cache.output_preview.clone();
3569 let envelope = append_bash_output_envelope(&cache, &mut output_preview);
3570 snapshot.output_preview = output_preview;
3571 snapshot.bash_output_list_envelope = envelope;
3572 snapshot.output_truncated = cache.output_truncated;
3573 }
3574 }
3575
3576 pub fn kill(&self, task_id: &str, session_id: &str) -> Result<BgTaskSnapshot, String> {
3577 self.kill_with_status(task_id, session_id, BgTaskStatus::Killed)
3578 }
3579
3580 pub fn kill_running_tasks_for_root(&self, project_root: &Path) -> usize {
3590 let canonical_root = canonicalized_path(project_root);
3591 let targets = self
3592 .inner
3593 .tasks
3594 .lock()
3595 .map(|tasks| {
3596 tasks
3597 .values()
3598 .filter_map(|task| {
3599 let state = task.state.lock().ok()?;
3600 let status = &state.metadata.status;
3601 let running = matches!(status, BgTaskStatus::Running)
3602 || (state.metadata.mode == BgMode::Pty
3603 && matches!(status, BgTaskStatus::Killing));
3604 if !running {
3605 return None;
3606 }
3607 let task_root = state
3608 .metadata
3609 .project_root
3610 .as_deref()
3611 .unwrap_or(&state.metadata.workdir);
3612 (canonicalized_path(task_root) == canonical_root)
3613 .then(|| (task.task_id.clone(), task.session_id.clone()))
3614 })
3615 .collect::<Vec<_>>()
3616 })
3617 .unwrap_or_default();
3618
3619 let mut killed = 0;
3620 for (task_id, session_id) in targets {
3621 match self.kill_with_status_reason(
3622 &task_id,
3623 &session_id,
3624 BgTaskStatus::Killed,
3625 Some(ROOT_RECLAIMED_REASON.to_string()),
3626 ) {
3627 Ok(_) => killed += 1,
3628 Err(error) => crate::slog_warn!(
3629 "failed to terminate background task {task_id} for reclaimed root {}: {error}",
3630 project_root.display()
3631 ),
3632 }
3633 }
3634 killed
3635 }
3636
3637 pub fn promote(&self, task_id: &str, session_id: &str) -> Result<bool, String> {
3638 let task = self
3639 .task_for_session(task_id, session_id)
3640 .ok_or_else(|| format!("background task not found: {task_id}"))?;
3641 let terminal_after_promote = {
3642 let mut state = task
3643 .state
3644 .lock()
3645 .map_err(|_| "background task lock poisoned".to_string())?;
3646 let updated = self
3647 .update_task_metadata(&task.paths, |metadata| {
3648 metadata.notify_on_completion = true;
3649 metadata.completion_delivered = false;
3650 })
3651 .map_err(|e| format!("failed to promote background task: {e}"))?;
3652 state.metadata = updated;
3653 state.metadata.status.is_terminal()
3654 };
3655 if terminal_after_promote {
3656 self.post_terminal_transition(&task, true)?;
3657 }
3658 Ok(true)
3659 }
3660
3661 pub(crate) fn kill_for_timeout(&self, task_id: &str, session_id: &str) -> Result<(), String> {
3662 self.kill_with_status(task_id, session_id, BgTaskStatus::TimedOut)
3663 .map(|_| ())
3664 }
3665
3666 pub fn cleanup_finished(&self, older_than: Duration) {
3667 let cutoff = Instant::now().checked_sub(older_than);
3668 let removable_paths: Vec<(String, TaskPaths)> =
3669 if let Ok(mut tasks) = self.inner.tasks.lock() {
3670 let removable = tasks
3671 .iter()
3672 .filter_map(|(task_id, task)| {
3673 let delivered_terminal = task
3674 .state
3675 .lock()
3676 .map(|state| {
3677 state.metadata.status.is_terminal()
3678 && state.metadata.completion_delivered
3679 })
3680 .unwrap_or(false);
3681 if !delivered_terminal {
3682 return None;
3683 }
3684
3685 let terminal_at = task.terminal_at.lock().ok().and_then(|at| *at);
3686 let expired = match (terminal_at, cutoff) {
3687 (Some(terminal_at), Some(cutoff)) => terminal_at <= cutoff,
3688 (Some(_), None) => true,
3689 (None, _) => false,
3690 };
3691 expired.then(|| task_id.clone())
3692 })
3693 .collect::<Vec<_>>();
3694
3695 removable
3696 .into_iter()
3697 .filter_map(|task_id| {
3698 tasks
3699 .remove(&task_id)
3700 .map(|task| (task_id, task.paths.clone()))
3701 })
3702 .collect()
3703 } else {
3704 Vec::new()
3705 };
3706
3707 for (task_id, paths) in removable_paths {
3708 match delete_task_bundle(&paths) {
3709 Ok(()) => log::debug!("deleted persisted background task bundle {task_id}"),
3710 Err(error) => crate::slog_warn!(
3711 "failed to delete persisted background task bundle {task_id}: {error}"
3712 ),
3713 }
3714 }
3715 }
3716
3717 pub fn drain_completions(&self) -> Vec<BgCompletion> {
3718 self.drain_completions_for_session(None)
3719 }
3720
3721 pub fn drain_completions_for_session(&self, session_id: Option<&str>) -> Vec<BgCompletion> {
3722 if let Some(session_id) = session_id {
3723 let pending_matches = self.pending_pattern_matches_for_session(session_id);
3724 if let Ok(sender) = self
3725 .inner
3726 .progress_sender
3727 .lock()
3728 .map(|sender| sender.clone())
3729 {
3730 if let Some(sender) = sender.as_ref() {
3731 for pattern_match in pending_matches {
3732 sender(PushFrame::BashPatternMatch(pattern_match));
3733 }
3734 }
3735 }
3736 }
3737 let completions = match self.inner.completions.lock() {
3738 Ok(completions) => completions,
3739 Err(_) => return Vec::new(),
3740 };
3741
3742 completions
3743 .iter()
3744 .filter(|completion| completion_matches_session(completion, session_id))
3745 .cloned()
3746 .collect()
3747 }
3748
3749 pub fn has_completions_for_session(&self, session_id: Option<&str>) -> bool {
3750 match self.inner.completions.lock() {
3751 Ok(completions) => completions
3752 .iter()
3753 .any(|completion| completion_matches_session(completion, session_id)),
3754 Err(_) => true,
3758 }
3759 }
3760
3761 pub fn unacked_wake_keys(&self) -> HashSet<String> {
3762 let mut keys = self
3763 .inner
3764 .completions
3765 .lock()
3766 .map(|completions| {
3767 completions
3768 .iter()
3769 .map(|completion| {
3770 format!(
3771 "completion\0{}\0{}",
3772 completion.session_id, completion.task_id
3773 )
3774 })
3775 .collect::<HashSet<_>>()
3776 })
3777 .unwrap_or_default();
3778 let Some((harness, pool)) = self.db_harness_and_pool() else {
3779 return keys;
3780 };
3781 let Ok(conn) = pool.lock() else {
3782 return keys;
3783 };
3784 if let Ok(rows) = crate::db::bash_watches::list_bash_pattern_watches(&conn, &harness) {
3785 keys.extend(rows.into_iter().filter(|row| row.pending_match).map(|row| {
3786 format!(
3787 "match\0{}\0{}\0{}",
3788 row.session_id, row.task_id, row.watch_id
3789 )
3790 }));
3791 }
3792 keys
3793 }
3794
3795 pub fn unacked_wake_count_for_session(&self, session_id: Option<&str>) -> usize {
3796 let completion_count = self
3797 .inner
3798 .completions
3799 .lock()
3800 .map(|completions| {
3801 completions
3802 .iter()
3803 .filter(|completion| completion_matches_session(completion, session_id))
3804 .count()
3805 })
3806 .unwrap_or(1);
3807 let Some((harness, pool)) = self.db_harness_and_pool() else {
3808 return completion_count;
3809 };
3810 let Ok(conn) = pool.lock() else {
3811 return completion_count.saturating_add(1);
3812 };
3813 let pending_matches = match session_id {
3814 Some(session_id) => {
3815 crate::db::bash_watches::count_pending_bash_pattern_watches_for_session(
3816 &conn, &harness, session_id,
3817 )
3818 }
3819 None => crate::db::bash_watches::count_pending_bash_pattern_watches(&conn, &harness),
3820 };
3821 completion_count.saturating_add(pending_matches.unwrap_or(1))
3822 }
3823
3824 pub fn has_unacked_wakes_for_session(&self, session_id: &str) -> bool {
3825 self.unacked_wake_count_for_session(Some(session_id)) > 0
3826 }
3827
3828 pub fn stuck_pending_watches_for_session(
3829 &self,
3830 session_id: &str,
3831 older_than: Duration,
3832 ) -> Vec<(String, String, u64)> {
3833 let Some((harness, pool)) = self.db_harness_and_pool() else {
3834 return Vec::new();
3835 };
3836 let Ok(conn) = pool.lock() else {
3837 return Vec::new();
3838 };
3839 let now = unix_millis();
3840 let minimum_age_ms = older_than.as_millis().min(u128::from(u64::MAX)) as u64;
3841 crate::db::bash_watches::list_bash_pattern_watches_for_session(&conn, &harness, session_id)
3842 .unwrap_or_default()
3843 .into_iter()
3844 .filter(|row| row.pending_match && self.task(&row.task_id).is_some())
3845 .filter_map(|row| {
3846 let created_at = u64::try_from(row.created_at).ok()?;
3847 let age_ms = now.saturating_sub(created_at);
3848 (age_ms >= minimum_age_ms).then_some((row.task_id, row.watch_id, age_ms))
3849 })
3850 .collect()
3851 }
3852
3853 pub fn ack_completions_for_session(
3854 &self,
3855 session_id: Option<&str>,
3856 task_ids: &[String],
3857 ) -> Vec<String> {
3858 if task_ids.is_empty() {
3859 return Vec::new();
3860 }
3861 let requested_task_ids = task_ids.iter().map(String::as_str).collect::<HashSet<_>>();
3862 let mut completion_sessions = HashMap::new();
3863 if let Ok(mut completions) = self.inner.completions.lock() {
3864 completions.retain(|completion| {
3865 let session_matches = session_id
3866 .map(|session_id| completion.session_id == session_id)
3867 .unwrap_or(true);
3868 if session_matches && requested_task_ids.contains(completion.task_id.as_str()) {
3869 completion_sessions
3870 .insert(completion.task_id.clone(), completion.session_id.clone());
3871 false
3872 } else {
3873 true
3874 }
3875 });
3876 }
3877
3878 let mut delivered = Vec::new();
3879 for task_id in task_ids {
3880 if self.has_erased_watch_reference(task_id) {
3881 if let Some((harness, pool)) = self.db_harness_and_pool() {
3882 if let Ok(conn) = pool.lock() {
3883 if let Ok(rows) =
3884 crate::db::bash_watches::list_bash_pattern_watches_by_task_id(
3885 &conn, &harness, task_id,
3886 )
3887 {
3888 for row in rows {
3889 let _ = crate::db::bash_watches::delete_bash_pattern_watch(
3890 &conn,
3891 &harness,
3892 &row.session_id,
3893 task_id,
3894 &row.watch_id,
3895 );
3896 }
3897 }
3898 }
3899 }
3900 self.clear_task_watch_state(task_id);
3901 delivered.push(task_id.clone());
3904 continue;
3905 }
3906 let task = if let Some(session_id) = session_id {
3907 self.task_for_session(task_id, session_id).or_else(|| {
3908 completion_sessions
3909 .contains_key(task_id)
3910 .then(|| self.task(task_id))
3911 .flatten()
3912 })
3913 } else if let Some(completion_session_id) = completion_sessions.get(task_id) {
3914 self.task_for_session(task_id, completion_session_id)
3915 .or_else(|| self.task(task_id))
3916 } else {
3917 self.task(task_id)
3918 };
3919 if let Some(task) = task {
3920 let terminal = task
3921 .state
3922 .lock()
3923 .map(|state| state.metadata.status.is_terminal())
3924 .unwrap_or(false);
3925 self.ack_persisted_watches_for_task(&task.session_id, task_id, terminal);
3929 if terminal {
3930 self.clear_task_watch_state(task_id);
3931 if task.set_completion_delivered(true, self).is_ok() {
3932 delivered.push(task_id.clone());
3933 }
3934 } else {
3935 self.sync_memory_watches_from_persistence(task_id);
3938 delivered.push(task_id.clone());
3939 }
3940 } else if let Some(session_id) = session_id {
3941 let terminal = self
3945 .persisted_task_is_terminal(session_id, task_id)
3946 .unwrap_or(true);
3947 self.ack_persisted_watches_for_task(session_id, task_id, terminal);
3948 if terminal {
3949 self.mark_persisted_completion_delivered(session_id, task_id);
3950 }
3951 delivered.push(task_id.clone());
3952 }
3953 }
3954
3955 delivered
3956 }
3957
3958 fn persisted_task_is_terminal(&self, session_id: &str, task_id: &str) -> Option<bool> {
3959 let (harness, pool) = self.db_harness_and_pool()?;
3960 let conn = pool.lock().ok()?;
3961 let row = crate::db::bash_tasks::get_bash_task(&conn, &harness, session_id, task_id)
3962 .ok()
3963 .flatten()?;
3964 Some(matches!(
3965 row.status.as_str(),
3966 "completed" | "failed" | "killed" | "timed_out" | "fate_unknown"
3967 ))
3968 }
3969
3970 fn mark_persisted_completion_delivered(&self, session_id: &str, task_id: &str) {
3971 let Some((harness, pool)) = self.db_harness_and_pool() else {
3972 return;
3973 };
3974 let Ok(conn) = pool.lock() else {
3975 return;
3976 };
3977 let _ = conn.execute(
3978 "UPDATE bash_tasks SET completion_delivered = 1
3979 WHERE harness = ?1 AND session_id = ?2 AND task_id = ?3
3980 AND status IN ('completed', 'failed', 'killed', 'timed_out', 'fate_unknown')",
3981 rusqlite::params![harness, session_id, task_id],
3982 );
3983 }
3984
3985 fn sync_memory_watches_from_persistence(&self, task_id: &str) {
3986 let Some((harness, pool)) = self.db_harness_and_pool() else {
3987 return;
3988 };
3989 let session_id = match self.task(task_id) {
3990 Some(task) => task.session_id.clone(),
3991 None => return,
3992 };
3993 let Ok(conn) = pool.lock() else {
3994 return;
3995 };
3996 let Ok(rows) = crate::db::bash_watches::list_bash_pattern_watches_for_task(
3997 &conn,
3998 &harness,
3999 &session_id,
4000 task_id,
4001 ) else {
4002 return;
4003 };
4004 let has_pending_match = rows.iter().any(|row| row.pending_match);
4005 let remaining: HashSet<String> = rows.into_iter().map(|row| row.watch_id).collect();
4006 if let Ok(mut registry) = self.inner.watch_registry.lock() {
4007 registry.reconcile_watch_ids(task_id, &remaining, has_pending_match);
4008 }
4009 }
4010
4011 pub fn pending_completions_for_session(&self, session_id: &str) -> Vec<BgCompletion> {
4012 self.inner
4013 .completions
4014 .lock()
4015 .map(|completions| {
4016 completions
4017 .iter()
4018 .filter(|completion| completion.session_id == session_id)
4019 .cloned()
4020 .collect()
4021 })
4022 .unwrap_or_default()
4023 }
4024
4025 fn remove_pending_completion(&self, task_id: &str) -> Option<BgCompletion> {
4026 let mut completions = self.inner.completions.lock().ok()?;
4027 let idx = completions
4028 .iter()
4029 .position(|completion| completion.task_id == task_id)?;
4030 completions.remove(idx)
4031 }
4032
4033 fn completion_snapshot_for_task(&self, task: &Arc<BgTask>) -> Option<BgCompletion> {
4034 let snapshot = self.snapshot_with_terminal_cache(task, RUNNING_OUTPUT_PREVIEW_BYTES);
4035 if !snapshot.info.status.is_terminal() {
4036 return None;
4037 }
4038 let (mut output_preview, output_truncated, cache) = if snapshot.info.mode == BgMode::Pty {
4039 (String::new(), false, None)
4040 } else {
4041 self.ensure_terminal_output_cache(task)
4042 .map(|cache| {
4043 let (preview, truncated) =
4044 completion_preview_for_cache(&cache, snapshot.exit_code);
4045 (preview, truncated, Some(cache))
4046 })
4047 .unwrap_or_else(|| (String::new(), false, None))
4048 };
4049 let bash_output_list_envelope = cache
4050 .as_ref()
4051 .and_then(|cache| append_bash_output_envelope(cache, &mut output_preview));
4052 Some(BgCompletion {
4053 task_id: snapshot.info.task_id,
4054 session_id: task.session_id.clone(),
4055 status: snapshot.info.status,
4056 exit_code: snapshot.exit_code,
4057 command: snapshot.info.command,
4058 output_preview,
4059 bash_output_list_envelope,
4060 output_truncated,
4061 original_tokens: None,
4062 compressed_tokens: None,
4063 tokens_skipped: false,
4064 status_reason: snapshot.info.status_reason,
4065 live_descendants: snapshot.live_descendants,
4066 live_descendants_omitted: snapshot.live_descendants_omitted,
4067 live_descendants_summary: snapshot.live_descendants_summary,
4068 })
4069 }
4070
4071 pub fn detach(&self) {
4072 self.inner.shutdown.store(true, Ordering::SeqCst);
4073 if let Ok(mut tasks) = self.inner.tasks.lock() {
4074 for task in tasks.values() {
4075 if let Ok(mut state) = task.state.lock() {
4076 match &mut state.runtime {
4077 TaskRuntime::Piped(child) => *child = None,
4078 TaskRuntime::Pty(runtime) => *runtime = None,
4079 }
4080 state.detached = true;
4081 }
4082 }
4083 tasks.clear();
4084 }
4085 }
4086
4087 pub fn shutdown(&self) {
4088 let tasks = self
4089 .inner
4090 .tasks
4091 .lock()
4092 .map(|tasks| {
4093 tasks
4094 .values()
4095 .map(|task| (task.task_id.clone(), task.session_id.clone()))
4096 .collect::<Vec<_>>()
4097 })
4098 .unwrap_or_default();
4099 for (task_id, session_id) in tasks {
4100 let _ = self.kill(&task_id, &session_id);
4101 }
4102 }
4103
4104 pub(crate) fn poll_task(&self, task: &Arc<BgTask>) -> Result<(), String> {
4105 if let Ok(state) = task.state.lock() {
4106 if let TaskRuntime::Pty(Some(pty)) = &state.runtime {
4107 if !pty.exit_observed.load(Ordering::SeqCst) {
4115 return Ok(());
4116 }
4117 }
4118 }
4119 let marker = match read_exit_marker(&task.paths) {
4120 Ok(Some(marker)) => marker,
4121 Ok(None) => return Ok(()),
4122 Err(error) => return Err(format!("failed to read exit marker: {error}")),
4123 };
4124 self.finalize_from_marker(task, marker, None)
4125 }
4126
4127 pub(crate) fn reap_child(&self, task: &Arc<BgTask>) {
4128 let mut needs_completion = false;
4129 {
4130 let Ok(mut state) = task.state.lock() else {
4131 return;
4132 };
4133 match &mut state.runtime {
4134 TaskRuntime::Piped(child_slot) => {
4135 if let Some(child) = child_slot.as_mut() {
4136 if let Ok(Some(status)) = child.try_wait() {
4137 *child_slot = None;
4138 state.detached = true;
4139 state.child_exit_observed = true;
4140 if let Some(handles) = state.io_handles.as_mut() {
4141 if handles.artifact_len(TaskArtifact::Exit).unwrap_or(1) == 0 {
4142 let marker = status
4143 .code()
4144 .map(|code| code.to_string())
4145 .unwrap_or_else(|| "1".to_string());
4146 let _ = handles.write(TaskArtifact::Exit, marker.as_bytes());
4147 }
4148 }
4149 }
4150 } else if state.detached {
4151 let child_known_dead = state.child_exit_observed
4152 || state
4153 .metadata
4154 .child_pid
4155 .is_some_and(|pid| !is_process_alive(pid));
4156 if child_known_dead {
4157 needs_completion =
4158 self.fail_without_exit_marker_if_needed(task, &mut state);
4159 }
4160 }
4161 }
4162 TaskRuntime::Pty(Some(pty)) => {
4163 if pty.exit_observed.load(Ordering::SeqCst) {
4164 drop(state);
4165 let _ = self.poll_task(task);
4166 return;
4167 }
4168 }
4169 TaskRuntime::Pty(None) => {}
4170 }
4171 }
4172 if needs_completion {
4173 let _ = self.post_terminal_transition(task, true);
4174 }
4175 }
4176
4177 fn fail_without_exit_marker_if_needed(
4178 &self,
4179 task: &Arc<BgTask>,
4180 state: &mut BgTaskState,
4181 ) -> bool {
4182 if state.metadata.status.is_terminal() {
4183 return false;
4184 }
4185 if matches!(read_exit_marker(&task.paths), Ok(Some(_))) {
4186 return false;
4187 }
4188 let child_exit_observed = state.child_exit_observed;
4189 let updated = self.update_task_metadata(&task.paths, |metadata| {
4190 let (status, reason) = if child_exit_observed {
4191 (
4192 BgTaskStatus::Failed,
4193 "process exited without exit marker".to_string(),
4194 )
4195 } else {
4196 (
4197 BgTaskStatus::FateUnknown,
4198 restart_fate_unknown_reason(metadata, &task.paths),
4199 )
4200 };
4201 metadata.mark_terminal(status, None, Some(reason));
4202 });
4203 if let Ok(metadata) = updated {
4204 state.pending_terminal_override = None;
4205 state.metadata = metadata;
4206 task.mark_terminal_now();
4207 return true;
4208 }
4209 false
4210 }
4211
4212 pub(crate) fn running_tasks(&self) -> Vec<Arc<BgTask>> {
4213 self.inner
4214 .tasks
4215 .lock()
4216 .map(|tasks| {
4217 tasks
4218 .values()
4219 .filter(|task| task.is_running())
4220 .cloned()
4221 .collect()
4222 })
4223 .unwrap_or_default()
4224 }
4225
4226 fn insert_rehydrated_task(
4227 &self,
4228 metadata: PersistedTask,
4229 paths: TaskPaths,
4230 detached: bool,
4231 ) -> Result<(), String> {
4232 let task_id = metadata.task_id.clone();
4233 let session_id = metadata.session_id.clone();
4234 let started = started_instant_from_unix_millis(metadata.started_at);
4235 let suppress_replayed_running_reminder = metadata.status == BgTaskStatus::Running;
4236 let mode = metadata.mode.clone();
4237 let task = Arc::new(BgTask {
4238 task_id: task_id.clone(),
4239 session_id,
4240 paths: paths.clone(),
4241 artifact_root: canonical_artifact_root(&paths),
4242 started,
4243 last_reminder_at: Mutex::new(suppress_replayed_running_reminder.then(Instant::now)),
4244 terminal_at: Mutex::new(metadata.status.is_terminal().then(Instant::now)),
4245 state: Mutex::new(BgTaskState {
4246 metadata,
4247 runtime: if mode == BgMode::Pty {
4248 TaskRuntime::Pty(None)
4249 } else {
4250 TaskRuntime::Piped(None)
4251 },
4252 io_handles: None,
4253 detached,
4254 child_exit_observed: false,
4261 descendant_sampling_started: false,
4262 buffer: BgBuffer::registered(&paths, mode.clone()),
4263 terminal_output_cache: None,
4264 pending_terminal_override: None,
4265 }),
4266 });
4267 self.record_live_delivery_session(&task.session_id);
4268 self.inner
4269 .tasks
4270 .lock()
4271 .map_err(|_| "background task registry lock poisoned".to_string())?
4272 .insert(task_id.clone(), Arc::clone(&task));
4273 self.rearm_persisted_watches(&task);
4277 Ok(())
4278 }
4279
4280 fn rearm_persisted_watches(&self, task: &Arc<BgTask>) {
4281 let Some((harness, pool)) = self.db_harness_and_pool() else {
4282 return;
4283 };
4284 let rows = {
4285 let Ok(conn) = pool.lock() else {
4286 return;
4287 };
4288 match crate::db::bash_watches::list_bash_pattern_watches_for_task(
4289 &conn,
4290 &harness,
4291 &task.session_id,
4292 &task.task_id,
4293 ) {
4294 Ok(rows) if !rows.is_empty() => rows,
4295 _ => return,
4296 }
4297 };
4298
4299 let mode = match task.state.lock() {
4300 Ok(state) => state.metadata.mode.clone(),
4301 Err(_) => return,
4302 };
4303 let terminal = task
4304 .state
4305 .lock()
4306 .map(|state| state.metadata.status.is_terminal())
4307 .unwrap_or(false);
4308 let completion_delivered = task
4309 .state
4310 .lock()
4311 .map(|state| state.metadata.completion_delivered)
4312 .unwrap_or(true);
4313
4314 let mut stdout = (mode == BgMode::Pipes)
4315 .then(|| open_task_artifact(&task.paths, TaskArtifact::Stdout))
4316 .transpose()
4317 .ok()
4318 .flatten();
4319 let mut stderr = (mode == BgMode::Pipes)
4320 .then(|| open_task_artifact(&task.paths, TaskArtifact::Stderr))
4321 .transpose()
4322 .ok()
4323 .flatten();
4324 let mut pty = (mode == BgMode::Pty)
4325 .then(|| open_task_artifact(&task.paths, TaskArtifact::Pty))
4326 .transpose()
4327 .ok()
4328 .flatten();
4329
4330 let mut pending_to_emit = Vec::new();
4331 let mut gap_matches = Vec::new();
4332 {
4333 let Ok(mut registry) = self.inner.watch_registry.lock() else {
4334 return;
4335 };
4336 let stdout_key = format!("{}:stdout", task.task_id);
4337 let stderr_key = format!("{}:stderr", task.task_id);
4338 let pty_key = format!("{}:pty", task.task_id);
4339
4340 let first = &rows[0];
4342 match mode {
4343 BgMode::Pipes => {
4344 registry.set_file_cursor(&stdout_key, first.stdout_offset.max(0) as u64);
4345 registry.set_file_cursor(&stderr_key, first.stderr_offset.max(0) as u64);
4346 }
4347 BgMode::Pty => {
4348 registry.set_file_cursor(&pty_key, first.pty_offset.max(0) as u64);
4349 }
4350 }
4351
4352 for row in &rows {
4353 let Ok(pattern) = WatchPattern::from_persisted(&row.pattern_kind, &row.pattern)
4354 else {
4355 crate::slog_warn!(
4356 "skipping unreadable persisted watch {}/{}",
4357 row.task_id,
4358 row.watch_id
4359 );
4360 continue;
4361 };
4362 if let Err(error) = registry.restore(
4363 row.watch_id.clone(),
4364 row.task_id.clone(),
4365 pattern,
4366 row.once,
4367 row.scanning,
4368 ) {
4369 crate::slog_warn!(
4370 "failed to restore watch {}/{}: {error}",
4371 row.task_id,
4372 row.watch_id
4373 );
4374 continue;
4375 }
4376 if row.pending_match {
4377 if let (Some(match_text), Some(match_offset), Some(context)) = (
4378 row.match_text.clone(),
4379 row.match_offset,
4380 row.match_context.clone(),
4381 ) {
4382 pending_to_emit.push(PatternMatch {
4383 watch_id: row.watch_id.clone(),
4384 task_id: row.task_id.clone(),
4385 match_text,
4386 match_offset: match_offset.max(0) as u64,
4387 context,
4388 once: row.once,
4389 });
4390 }
4391 }
4392 }
4393
4394 let should_gap_scan =
4398 rows.iter().any(|row| row.scanning) && !pending_to_emit.iter().any(|m| m.once);
4399 if should_gap_scan {
4400 match mode {
4401 BgMode::Pipes => {
4402 if let (Some(stdout), Some(stderr)) = (stdout.as_mut(), stderr.as_mut()) {
4403 gap_matches.extend(registry.scan_file_new_bytes(
4404 &stdout_key,
4405 &task.task_id,
4406 stdout,
4407 ));
4408 gap_matches.extend(registry.scan_file_new_bytes(
4409 &stderr_key,
4410 &task.task_id,
4411 stderr,
4412 ));
4413 }
4414 }
4415 BgMode::Pty => {
4416 if let Some(pty) = pty.as_mut() {
4417 gap_matches.extend(registry.scan_file_new_bytes(
4418 &pty_key,
4419 &task.task_id,
4420 pty,
4421 ));
4422 }
4423 }
4424 }
4425 }
4426 }
4427
4428 let (stdout_offset, stderr_offset, pty_offset) = self.watch_stream_cursors(&task.task_id);
4429 for pattern_match in &gap_matches {
4430 self.persist_watch_match(
4431 &task.session_id,
4432 &task.task_id,
4433 pattern_match,
4434 stdout_offset,
4435 stderr_offset,
4436 pty_offset,
4437 );
4438 }
4439 if !gap_matches.is_empty() || rows.iter().any(|row| row.scanning) {
4440 self.persist_task_watch_cursors(
4441 &task.session_id,
4442 &task.task_id,
4443 stdout_offset,
4444 stderr_offset,
4445 pty_offset,
4446 );
4447 }
4448
4449 let emitted_pending = !pending_to_emit.is_empty();
4451 let to_emit = if emitted_pending {
4452 pending_to_emit
4453 } else {
4454 gap_matches
4455 };
4456 for pattern_match in to_emit {
4457 self.emit_bash_pattern_match(&task.session_id, pattern_match);
4458 }
4459
4460 if !terminal {
4461 return;
4462 }
4463
4464 let (watch_controlled, watch_matched) = self.task_watch_state(&task.task_id);
4465 if !watch_controlled {
4466 return;
4467 }
4468 if watch_matched {
4469 let _ = self.remove_pending_completion(&task.task_id);
4472 return;
4473 }
4474 if completion_delivered {
4475 self.clear_task_watch_state(&task.task_id);
4477 self.delete_persisted_watches_for_task(&task.session_id, &task.task_id);
4478 return;
4479 }
4480 if let Some(completion) = self
4481 .remove_pending_completion(&task.task_id)
4482 .or_else(|| self.completion_snapshot_for_task(task))
4483 {
4484 self.record_bash_watch_exit(&completion, true);
4485 }
4486 self.clear_task_watch_state(&task.task_id);
4488 }
4489
4490 fn kill_with_status(
4491 &self,
4492 task_id: &str,
4493 session_id: &str,
4494 terminal_status: BgTaskStatus,
4495 ) -> Result<BgTaskSnapshot, String> {
4496 self.kill_with_status_reason(task_id, session_id, terminal_status, None)
4497 }
4498
4499 fn kill_with_status_reason(
4500 &self,
4501 task_id: &str,
4502 session_id: &str,
4503 terminal_status: BgTaskStatus,
4504 reason: Option<String>,
4505 ) -> Result<BgTaskSnapshot, String> {
4506 let task = self
4507 .task_for_session(task_id, session_id)
4508 .ok_or_else(|| format!("background task not found: {task_id}"))?;
4509 let mut terminalized = false;
4510 #[cfg_attr(not(unix), allow(unused_mut))]
4511 let mut kill_signaled = false;
4512 #[cfg_attr(not(unix), allow(unused_mut))]
4513 let mut kill_reached = 0;
4514
4515 {
4516 let mut state = task
4517 .state
4518 .lock()
4519 .map_err(|_| "background task lock poisoned".to_string())?;
4520 if state.metadata.status.is_terminal() {
4521 state.pending_terminal_override = None;
4522 #[cfg(unix)]
4523 let live_count = state
4524 .metadata
4525 .live_descendants
4526 .as_ref()
4527 .map(Vec::len)
4528 .unwrap_or(0)
4529 + state.metadata.live_descendants_omitted;
4530 #[cfg(unix)]
4531 if live_count > 0 {
4532 if let Some(pgid) = state.metadata.pgid {
4533 kill_signaled = true;
4534 kill_reached = live_count;
4535 terminate_pgid(pgid, None);
4536 let sample = live_process_group_members(pgid);
4537 state.metadata.live_descendants =
4538 sample.as_ref().map(|(members, _)| members.clone());
4539 state.metadata.live_descendants_omitted =
4540 sample.as_ref().map(|(_, omitted)| *omitted).unwrap_or(0);
4541 self.persist_task(&task.paths, &state.metadata)
4542 .map_err(|error| {
4543 format!("failed to persist post-kill descendant sample: {error}")
4544 })?;
4545 }
4546 }
4547 } else if let Ok(Some(marker)) = read_exit_marker(&task.paths) {
4548 state.metadata =
4549 terminal_metadata_from_marker(state.metadata.clone(), marker, reason.clone());
4550
4551 state.pending_terminal_override = None;
4552 task.mark_terminal_now();
4553 match &mut state.runtime {
4554 TaskRuntime::Piped(child_slot) => reap_piped_child(child_slot),
4561 TaskRuntime::Pty(runtime) => *runtime = None,
4562 }
4563 state.detached = true;
4564 self.persist_task(&task.paths, &state.metadata)
4565 .map_err(|e| format!("failed to persist terminal state: {e}"))?;
4566 terminalized = true;
4567 } else {
4568 let was_already_killing = state.metadata.status == BgTaskStatus::Killing;
4569 if !was_already_killing {
4570 state.metadata.status = BgTaskStatus::Killing;
4571 }
4572 if reason.is_some() {
4573 state.metadata.status_reason = reason.clone();
4574 }
4575 if !was_already_killing || reason.is_some() {
4576 self.persist_task(&task.paths, &state.metadata)
4577 .map_err(|e| format!("failed to persist killing state: {e}"))?;
4578 }
4579
4580 #[cfg(unix)]
4581 let pgid = state.metadata.pgid;
4582 #[cfg(windows)]
4583 let child_pid = state.metadata.child_pid;
4584 if !was_already_killing
4585 && state.metadata.mode == BgMode::Pty
4586 && terminal_status == BgTaskStatus::TimedOut
4587 {
4588 state.pending_terminal_override = Some(BgTaskStatus::TimedOut);
4589 }
4590
4591 #[cfg(windows)]
4592 let mut pty_forced_terminal_status: Option<BgTaskStatus> = None;
4593
4594 match &mut state.runtime {
4595 TaskRuntime::Piped(child_slot) => {
4596 #[cfg(unix)]
4597 if let Some(pgid) = pgid {
4598 terminate_pgid(pgid, child_slot.as_mut());
4599 }
4600 #[cfg(windows)]
4601 if let Some(child) = child_slot.as_mut() {
4602 super::process::terminate_process(child);
4603 } else if let Some(pid) = child_pid {
4604 terminate_pid(pid);
4605 }
4606 if let Some(child) = child_slot.as_mut() {
4607 let _ = child.wait();
4608 }
4609 *child_slot = None;
4610 state.detached = true;
4611
4612 if let Some(handles) = state.io_handles.as_mut() {
4613 match handles.write(TaskArtifact::Exit, b"killed") {
4614 Ok(()) => {}
4615 Err(error)
4616 if error.kind() == std::io::ErrorKind::Interrupted
4617 && error.to_string().contains(
4618 super::persistence::ARTIFACT_CONCURRENTLY_REPLACED,
4619 ) =>
4620 {
4621 write_kill_marker_if_absent(&task.paths).map_err(|e| {
4627 format!("failed to write kill marker after replace: {e}")
4628 })?;
4629 }
4630 Err(error) => {
4631 return Err(format!(
4632 "failed to write retained kill marker: {error}"
4633 ));
4634 }
4635 }
4636 } else {
4637 write_kill_marker_if_absent(&task.paths)
4638 .map_err(|e| format!("failed to write kill marker: {e}"))?;
4639 }
4640
4641 let exit_code = terminal_exit_code_for_status(&terminal_status);
4642 state
4643 .metadata
4644 .mark_terminal(terminal_status, exit_code, reason.clone());
4645
4646 state.pending_terminal_override = None;
4647 task.mark_terminal_now();
4648 self.persist_task(&task.paths, &state.metadata)
4649 .map_err(|e| format!("failed to persist killed state: {e}"))?;
4650 terminalized = true;
4651 }
4652 TaskRuntime::Pty(Some(pty)) => {
4653 pty.was_killed.store(true, Ordering::SeqCst);
4654 if let Err(error) = pty.killer.kill() {
4655 crate::slog_warn!(
4656 "[pty-kill] {task_id} ChildKiller::kill failed: {error}"
4657 );
4658 }
4659 if let Some(pid) = pty.child_pid {
4660 #[cfg(unix)]
4661 terminate_pgid(pid as i32, None);
4662 #[cfg(windows)]
4663 terminate_pid(pid);
4664 }
4665 drop(pty.master.take());
4666
4667 #[cfg(windows)]
4668 {
4669 let default_status = if terminal_status == BgTaskStatus::TimedOut {
4670 BgTaskStatus::TimedOut
4671 } else {
4672 BgTaskStatus::Killed
4673 };
4674 pty_forced_terminal_status = Some(
4675 state
4676 .pending_terminal_override
4677 .take()
4678 .unwrap_or(default_status),
4679 );
4680 }
4681 }
4682 TaskRuntime::Pty(None) => {}
4683 }
4684
4685 #[cfg(windows)]
4686 if let Some(target_status) = pty_forced_terminal_status {
4687 if !task.paths.exit.exists() {
4688 write_kill_marker_if_absent(&task.paths)
4689 .map_err(|e| format!("failed to write kill marker: {e}"))?;
4690 }
4691
4692 let exit_code = terminal_exit_code_for_status(&target_status);
4693 state
4694 .metadata
4695 .mark_terminal(target_status, exit_code, reason.clone());
4696
4697 state.pending_terminal_override = None;
4698 task.mark_terminal_now();
4699 if let TaskRuntime::Pty(runtime) = &mut state.runtime {
4700 *runtime = None;
4701 }
4702 state.detached = true;
4703 self.persist_task(&task.paths, &state.metadata)
4704 .map_err(|e| format!("failed to persist killed PTY state: {e}"))?;
4705 terminalized = true;
4706 }
4707 }
4708 }
4709
4710 if terminalized {
4711 self.post_terminal_transition(&task, true)?;
4712 }
4713 let mut snapshot = self.snapshot_with_terminal_cache(&task, RUNNING_OUTPUT_PREVIEW_BYTES);
4714 snapshot.kill_signaled = kill_signaled;
4715 snapshot.kill_reached = kill_reached;
4716 Ok(snapshot)
4717 }
4718
4719 fn finalize_from_marker(
4720 &self,
4721 task: &Arc<BgTask>,
4722 marker: ExitMarker,
4723 reason: Option<String>,
4724 ) -> Result<(), String> {
4725 let mut pty_reader_done = None;
4726 {
4727 let mut state = task
4728 .state
4729 .lock()
4730 .map_err(|_| "background task lock poisoned".to_string())?;
4731 if state.metadata.status.is_terminal() {
4732 state.pending_terminal_override = None;
4733 return Ok(());
4734 }
4735
4736 let pending_override = state.pending_terminal_override.take();
4737 let is_pty = state.metadata.mode == BgMode::Pty;
4738 let reason = reason.or_else(|| state.metadata.status_reason.clone());
4739 let updated = self
4740 .update_task_metadata(&task.paths, |metadata| {
4741 let new_metadata = if is_pty && marker == ExitMarker::Killed {
4742 let mut metadata = metadata.clone();
4743 let target_status = pending_override.unwrap_or(BgTaskStatus::Killed);
4744 let exit_code = terminal_exit_code_for_status(&target_status);
4745 metadata.mark_terminal(target_status, exit_code, reason.clone());
4746 metadata
4747 } else {
4748 terminal_metadata_from_marker(metadata.clone(), marker, reason.clone())
4749 };
4750 *metadata = new_metadata;
4751 })
4752 .map_err(|e| format!("failed to persist terminal state: {e}"))?;
4753 state.metadata = updated;
4754 task.mark_terminal_now();
4755 match &mut state.runtime {
4756 TaskRuntime::Piped(child_slot) => reap_piped_child(child_slot),
4761 TaskRuntime::Pty(runtime) => {
4762 pty_reader_done = runtime
4763 .as_ref()
4764 .map(|runtime| Arc::clone(&runtime.reader_done));
4765 *runtime = None;
4766 }
4767 }
4768 state.detached = true;
4769 }
4770
4771 if let Some(reader_done) = pty_reader_done {
4772 let deadline = Instant::now() + Duration::from_millis(200);
4773 while !reader_done.load(Ordering::SeqCst) && Instant::now() < deadline {
4774 std::thread::sleep(Duration::from_millis(10));
4775 }
4776 }
4777
4778 self.scan_task_watch_output(task);
4781
4782 self.post_terminal_transition(task, true)
4783 }
4784
4785 fn enqueue_completion_if_needed(
4786 &self,
4787 metadata: &PersistedTask,
4788 paths: Option<&TaskPaths>,
4789 emit_frame: bool,
4790 ) {
4791 if metadata.status.is_terminal() && !metadata.completion_delivered {
4792 let cache =
4793 paths.and_then(|paths| self.render_terminal_output_from_paths(metadata, paths));
4794 self.enqueue_completion_from_parts(metadata, None, paths, emit_frame, cache.as_ref());
4795 }
4796 }
4797
4798 fn render_terminal_output_from_paths(
4799 &self,
4800 metadata: &PersistedTask,
4801 paths: &TaskPaths,
4802 ) -> Option<TerminalOutputCache> {
4803 if metadata.mode == BgMode::Pty {
4804 return None;
4805 }
4806 let mut buffer = BgBuffer::registered(paths, BgMode::Pipes);
4807 let disk_truncation = buffer.enforce_terminal_cap();
4808 Some(self.render_terminal_output(metadata, &buffer, disk_truncation, Some(paths)))
4809 }
4810
4811 fn enqueue_completion_from_parts(
4812 &self,
4813 metadata: &PersistedTask,
4814 buffer: Option<&BgBuffer>,
4815 paths: Option<&TaskPaths>,
4816 emit_frame: bool,
4817 terminal_render: Option<&TerminalOutputCache>,
4818 ) {
4819 if !metadata.status.is_terminal() {
4830 return;
4831 }
4832
4833 let owned_buffer = if buffer.is_none() && metadata.mode != BgMode::Pty {
4834 paths.map(|paths| BgBuffer::registered(paths, BgMode::Pipes))
4835 } else {
4836 None
4837 };
4838 let render_buffer = buffer.or(owned_buffer.as_ref());
4839 let owned_render = if terminal_render.is_none() {
4840 render_buffer.map(|buffer| {
4841 let mut capped_buffer = buffer.clone();
4842 let disk_truncation = capped_buffer.enforce_terminal_cap();
4843 self.render_terminal_output(metadata, &capped_buffer, disk_truncation, paths)
4844 })
4845 } else {
4846 None
4847 };
4848 let render = terminal_render.or(owned_render.as_ref());
4849
4850 let (mut output_preview, output_truncated) = render
4854 .map(|cache| completion_preview_for_cache(cache, metadata.exit_code))
4855 .unwrap_or_else(|| (String::new(), false));
4856 if metadata.status == BgTaskStatus::FateUnknown {
4857 if let Some(reason) = metadata.status_reason.as_deref() {
4858 output_preview = if output_preview.is_empty() {
4859 reason.to_string()
4860 } else {
4861 format!("{reason}\n{output_preview}")
4862 };
4863 }
4864 }
4865 let bash_output_list_envelope =
4866 render.and_then(|cache| append_bash_output_envelope(cache, &mut output_preview));
4867
4868 let token_counts = self.completion_token_counts(
4869 metadata,
4870 buffer,
4871 paths,
4872 render.map(|render| render.output_preview.as_str()),
4873 );
4874 let completion = BgCompletion {
4875 task_id: metadata.task_id.clone(),
4876 session_id: metadata.session_id.clone(),
4877 status: metadata.status.clone(),
4878 exit_code: metadata.exit_code,
4879 command: metadata.command.clone(),
4880 output_preview,
4881 bash_output_list_envelope,
4882 output_truncated,
4883 original_tokens: token_counts.original_tokens,
4884 compressed_tokens: token_counts.compressed_tokens,
4885 tokens_skipped: token_counts.tokens_skipped,
4886 status_reason: metadata.status_reason.clone(),
4887 live_descendants: metadata.live_descendants.clone(),
4888 live_descendants_omitted: metadata.live_descendants_omitted,
4889 live_descendants_summary: live_descendants_summary(metadata),
4890 };
4891
4892 self.record_compression_event_if_applicable(metadata, &token_counts);
4903
4904 self.sync_memory_watches_from_persistence(&metadata.task_id);
4907 let (watch_controlled, watch_matched) = self.task_watch_state(&metadata.task_id);
4908 if watch_controlled {
4909 if !watch_matched && !metadata.completion_delivered {
4910 self.record_bash_watch_exit(&completion, emit_frame);
4911 }
4912 self.clear_task_watch_state(&metadata.task_id);
4915 return;
4916 }
4917
4918 if metadata.completion_delivered {
4928 return;
4929 }
4930
4931 let pushed = if let Ok(mut completions) = self.inner.completions.lock() {
4934 if completions
4935 .iter()
4936 .any(|existing| existing.task_id == metadata.task_id)
4937 {
4938 false
4939 } else {
4940 completions.push_back(completion.clone());
4941 true
4942 }
4943 } else {
4944 false
4945 };
4946
4947 if pushed && emit_frame {
4948 self.emit_bash_completed(completion);
4949 }
4950 }
4951
4952 fn record_compression_event_if_applicable(
4953 &self,
4954 metadata: &PersistedTask,
4955 token_counts: &CompletionTokenCounts,
4956 ) {
4957 if metadata.mode == BgMode::Pty {
4958 return;
4959 }
4960
4961 let (original_tokens, compressed_tokens, original_bytes, compressed_bytes) = match (
4962 token_counts.original_tokens,
4963 token_counts.compressed_tokens,
4964 token_counts.original_bytes,
4965 token_counts.compressed_bytes,
4966 ) {
4967 (
4968 Some(original_tokens),
4969 Some(compressed_tokens),
4970 Some(original_bytes),
4971 Some(compressed_bytes),
4972 ) => (
4973 original_tokens,
4974 compressed_tokens,
4975 original_bytes,
4976 compressed_bytes,
4977 ),
4978 _ => {
4979 crate::slog_warn!(
4980 "compression event skipped for {}: token counts unavailable (likely spill file missing or unreadable)",
4981 metadata.task_id
4982 );
4983 return;
4984 }
4985 };
4986
4987 let pool = self.inner.db_pool.read().ok().and_then(|slot| slot.clone());
4988 let Some(pool) = pool else {
4989 crate::slog_warn!(
4990 "compression event skipped for {}: db_pool not initialized — was configure run?",
4991 metadata.task_id
4992 );
4993 return;
4994 };
4995 let harness = self
4996 .inner
4997 .db_harness
4998 .read()
4999 .ok()
5000 .and_then(|slot| slot.clone());
5001 let Some(harness) = harness else {
5002 crate::slog_warn!(
5003 "compression event insert skipped for {}: harness not configured",
5004 metadata.task_id
5005 );
5006 return;
5007 };
5008
5009 let project_root = metadata
5010 .project_root
5011 .as_deref()
5012 .unwrap_or(&metadata.workdir);
5013 let project_key = crate::path_identity::project_scope_key(project_root);
5014 let row = crate::db::compression_events::CompressionEventRow {
5015 harness: &harness,
5016 session_id: Some(&metadata.session_id),
5017 project_key: &project_key,
5018 tool: "bash",
5019 task_id: Some(&metadata.task_id),
5020 command: Some(&metadata.command),
5021 compressor: if metadata.compressed {
5022 "registry"
5023 } else {
5024 "none"
5025 },
5026 original_bytes,
5027 compressed_bytes,
5028 original_tokens,
5029 compressed_tokens,
5030 created_at: unix_millis() as i64,
5031 };
5032
5033 let conn = match pool.lock() {
5034 Ok(conn) => conn,
5035 Err(_) => {
5036 crate::slog_warn!(
5037 "compression event insert failed for {}: db mutex poisoned",
5038 metadata.task_id
5039 );
5040 return;
5041 }
5042 };
5043 match crate::db::compression_events::insert_compression_event(&conn, &row) {
5044 Ok(Some(row_id)) => {
5045 self.inner
5049 .compression_aggregates
5050 .record_successful_insert(&conn, &row, row_id);
5051 crate::slog_debug!(
5055 "compression event recorded for {} (project={}, session={}, {} → {} tokens)",
5056 metadata.task_id,
5057 project_key,
5058 metadata.session_id,
5059 original_tokens,
5060 compressed_tokens
5061 );
5062 }
5063 Ok(None) => {
5064 crate::slog_debug!(
5065 "duplicate compression event ignored for {} (project={}, session={})",
5066 metadata.task_id,
5067 project_key,
5068 metadata.session_id
5069 );
5070 }
5071 Err(error) => {
5072 crate::slog_warn!(
5073 "compression event insert failed for {}: {}",
5074 metadata.task_id,
5075 error
5076 );
5077 }
5078 }
5079 }
5080
5081 fn emit_bash_pattern_match(&self, session_id: &str, pattern_match: PatternMatch) {
5082 if !self.originating_session_has_live_route(session_id) {
5083 return;
5084 }
5085 let Ok(progress_sender) = self
5086 .inner
5087 .progress_sender
5088 .lock()
5089 .map(|sender| sender.clone())
5090 else {
5091 return;
5092 };
5093 if let Some(sender) = progress_sender.as_ref() {
5094 let frame = if pattern_match.match_text == WATCH_TASK_EXIT_TEXT {
5095 BashPatternMatchFrame::task_exit(
5096 pattern_match.task_id,
5097 session_id.to_string(),
5098 pattern_match.match_text,
5099 pattern_match.context,
5100 )
5101 } else {
5102 BashPatternMatchFrame::new(
5103 pattern_match.task_id,
5104 session_id.to_string(),
5105 pattern_match.watch_id,
5106 pattern_match.match_text,
5107 pattern_match.match_offset,
5108 pattern_match.context,
5109 pattern_match.once,
5110 )
5111 };
5112 sender(PushFrame::BashPatternMatch(frame));
5113 }
5114 }
5115
5116 fn emit_bash_watch_erased(&self, session_id: &str, task_id: &str, watch_id: &str) {
5117 if !self.originating_session_has_live_route(session_id) {
5118 return;
5119 }
5120 let Ok(progress_sender) = self
5121 .inner
5122 .progress_sender
5123 .lock()
5124 .map(|sender| sender.clone())
5125 else {
5126 return;
5127 };
5128 let Some(sender) = progress_sender.as_ref() else {
5129 return;
5130 };
5131 sender(PushFrame::BashPatternMatch(
5132 BashPatternMatchFrame::watch_target_erased(
5133 task_id,
5134 session_id,
5135 watch_id,
5136 WATCH_TARGET_ERASED_TEXT,
5137 WATCH_TARGET_ERASED_CONTEXT,
5138 ),
5139 ));
5140 }
5141
5142 fn record_bash_watch_exit(&self, completion: &BgCompletion, emit_frame: bool) {
5143 let status = completion_status_text(&completion.status, completion.exit_code);
5144 let preview = completion.output_preview.trim_end();
5145 let context = if preview.is_empty() {
5146 format!("task {} exited ({status})", completion.task_id)
5147 } else {
5148 format!(
5149 "task {} exited ({status})
5150{preview}",
5151 completion.task_id
5152 )
5153 };
5154 let frame = BashPatternMatchFrame::task_exit(
5155 completion.task_id.clone(),
5156 completion.session_id.clone(),
5157 format!("exited ({status})"),
5158 context,
5159 );
5160 self.persist_bash_watch_exit(&frame);
5161 if emit_frame {
5162 self.emit_bash_watch_exit(frame);
5163 }
5164 }
5165
5166 fn persist_bash_watch_exit(&self, frame: &BashPatternMatchFrame) {
5167 let Some((harness, pool)) = self.db_harness_and_pool() else {
5168 return;
5169 };
5170 let Ok(conn) = pool.lock() else {
5171 return;
5172 };
5173 let Ok(mut rows) = crate::db::bash_watches::list_bash_pattern_watches_for_task(
5174 &conn,
5175 &harness,
5176 &frame.session_id,
5177 &frame.task_id,
5178 ) else {
5179 return;
5180 };
5181 let Some(mut row) = rows
5182 .iter()
5183 .position(|row| row.match_text.as_deref() == Some(WATCH_TASK_EXIT_TEXT))
5184 .map(|index| rows.swap_remove(index))
5185 .or_else(|| rows.into_iter().next())
5186 else {
5187 return;
5188 };
5189 row.scanning = false;
5190 row.pending_match = true;
5191 row.match_text = Some(WATCH_TASK_EXIT_TEXT.to_string());
5192 row.match_offset = Some(0);
5193 row.match_context = Some(frame.context.clone());
5194 if let Err(error) = crate::db::bash_watches::upsert_bash_pattern_watch(&conn, &row) {
5195 crate::slog_warn!(
5196 "persist bash watch task-exit failed for {}/{}: {error}",
5197 frame.task_id,
5198 row.watch_id
5199 );
5200 }
5201 }
5202
5203 fn emit_bash_watch_exit(&self, frame: BashPatternMatchFrame) {
5204 if !self.originating_session_has_live_route(&frame.session_id) {
5205 return;
5206 }
5207 let Ok(progress_sender) = self
5208 .inner
5209 .progress_sender
5210 .lock()
5211 .map(|sender| sender.clone())
5212 else {
5213 return;
5214 };
5215 let Some(sender) = progress_sender.as_ref() else {
5216 return;
5217 };
5218 sender(PushFrame::BashPatternMatch(frame));
5219 }
5220
5221 fn emit_bash_completed(&self, completion: BgCompletion) {
5222 if !self.originating_session_has_live_route(&completion.session_id) {
5223 return;
5224 }
5225 let Ok(progress_sender) = self
5226 .inner
5227 .progress_sender
5228 .lock()
5229 .map(|sender| sender.clone())
5230 else {
5231 return;
5232 };
5233 let Some(sender) = progress_sender.as_ref() else {
5234 return;
5235 };
5236 let mut frame = BashCompletedFrame::new(
5244 completion.task_id,
5245 completion.session_id,
5246 completion.status,
5247 completion.exit_code,
5248 completion.command,
5249 completion.output_preview,
5250 completion.output_truncated,
5251 completion.original_tokens,
5252 completion.compressed_tokens,
5253 completion.tokens_skipped,
5254 );
5255 frame.bash_output_list_envelope = completion.bash_output_list_envelope;
5256 frame.status_reason = completion.status_reason;
5257 frame.live_descendants = completion.live_descendants;
5258 frame.live_descendants_omitted = completion.live_descendants_omitted;
5259 frame.live_descendants_summary = completion.live_descendants_summary;
5260 sender(PushFrame::BashCompleted(frame));
5261 }
5262
5263 fn completion_token_counts(
5264 &self,
5265 metadata: &PersistedTask,
5266 buffer: Option<&BgBuffer>,
5267 paths: Option<&TaskPaths>,
5268 rendered_output: Option<&str>,
5269 ) -> CompletionTokenCounts {
5270 if metadata.mode == BgMode::Pty {
5271 return CompletionTokenCounts::skipped();
5272 }
5273
5274 let raw = match buffer {
5275 Some(buffer) => buffer.read_for_token_count(TOKENIZE_CAP_BYTES_PER_STREAM),
5276 None => paths
5277 .map(|paths| {
5278 read_for_token_count_from_disk(metadata, paths, TOKENIZE_CAP_BYTES_PER_STREAM)
5279 })
5280 .unwrap_or(TokenCountInput::Skipped),
5281 };
5282
5283 let TokenCountInput::Text(raw_output) = raw else {
5284 return CompletionTokenCounts::skipped();
5285 };
5286
5287 let original_tokens = token_count_u32(&raw_output);
5288 let original_bytes = raw_output.len() as i64;
5289 let compressed_output = rendered_output.unwrap_or(&raw_output);
5290 let compressed_tokens = token_count_u32(compressed_output);
5291 let compressed_bytes = compressed_output.len() as i64;
5292 CompletionTokenCounts {
5293 original_tokens: Some(original_tokens),
5294 compressed_tokens: Some(compressed_tokens),
5295 original_bytes: Some(original_bytes),
5296 compressed_bytes: Some(compressed_bytes),
5297 tokens_skipped: false,
5298 }
5299 }
5300
5301 pub(crate) fn maybe_emit_long_running_reminder(&self, task: &Arc<BgTask>) {
5302 if !self
5303 .inner
5304 .long_running_reminder_enabled
5305 .load(Ordering::SeqCst)
5306 {
5307 return;
5308 }
5309 let interval_ms = self
5310 .inner
5311 .long_running_reminder_interval_ms
5312 .load(Ordering::SeqCst);
5313 if interval_ms == 0 {
5314 return;
5315 }
5316 let interval = Duration::from_millis(interval_ms);
5317 let now = Instant::now();
5318 let Ok(mut last_reminder_at) = task.last_reminder_at.lock() else {
5319 return;
5320 };
5321 let since = last_reminder_at.unwrap_or(task.started);
5322 if now.duration_since(since) < interval {
5323 return;
5324 }
5325 let command = task
5326 .state
5327 .lock()
5328 .map(|state| state.metadata.command.clone())
5329 .unwrap_or_default();
5330 *last_reminder_at = Some(now);
5331 self.emit_bash_long_running(BashLongRunningFrame::new(
5332 task.task_id.clone(),
5333 task.session_id.clone(),
5334 command,
5335 task.started.elapsed().as_millis() as u64,
5336 ));
5337 }
5338
5339 fn emit_bash_long_running(&self, frame: BashLongRunningFrame) {
5340 if !self.originating_session_has_live_route(&frame.session_id) {
5341 return;
5342 }
5343 let Ok(progress_sender) = self
5344 .inner
5345 .progress_sender
5346 .lock()
5347 .map(|sender| sender.clone())
5348 else {
5349 return;
5350 };
5351 if let Some(sender) = progress_sender.as_ref() {
5352 sender(PushFrame::BashLongRunning(frame));
5353 }
5354 }
5355
5356 fn task(&self, task_id: &str) -> Option<Arc<BgTask>> {
5357 validate_task_id(task_id).ok()?;
5358 self.inner
5359 .tasks
5360 .lock()
5361 .ok()
5362 .and_then(|tasks| tasks.get(task_id).cloned())
5363 }
5364
5365 fn task_for_session(&self, task_id: &str, session_id: &str) -> Option<Arc<BgTask>> {
5366 self.task(task_id)
5367 .filter(|task| task.session_id == session_id)
5368 }
5369
5370 pub fn try_health_counts(&self) -> Option<BgTaskHealthCounts> {
5371 let running = self
5372 .inner
5373 .tasks
5374 .try_lock()
5375 .ok()
5376 .map(|tasks| tasks.values().filter(|task| task.is_running()).count())?;
5377 let pending_completions = self.inner.completions.try_lock().ok().map(|q| q.len())?;
5378 Some(BgTaskHealthCounts {
5379 running,
5380 pending_completions,
5381 })
5382 }
5383
5384 pub(crate) fn detached_live_process_count(&self) -> usize {
5387 let Some(pids) = self.inner.tasks.try_lock().ok().map(|tasks| {
5388 tasks
5389 .values()
5390 .filter_map(|task| {
5391 task.state
5392 .try_lock()
5393 .ok()
5394 .and_then(|state| state.metadata.child_pid)
5395 })
5396 .collect::<Vec<_>>()
5397 }) else {
5398 return 0;
5399 };
5400 pids.into_iter()
5401 .filter(|pid| is_process_alive(*pid))
5402 .count()
5403 }
5404
5405 pub fn estimated_memory(&self) -> crate::memory::MemoryEstimate {
5409 let tasks = match self.inner.tasks.try_lock() {
5410 Ok(tasks) => tasks.values().cloned().collect::<Vec<_>>(),
5411 Err(_) => return crate::memory::MemoryEstimate::busy(),
5412 };
5413 let mut bytes = 0u64;
5414 let mut terminal_output_caches = 0usize;
5415 let mut sessions = HashSet::new();
5416 for task in &tasks {
5417 sessions.insert(task.session_id.clone());
5418 let state = match task.state.try_lock() {
5419 Ok(state) => state,
5420 Err(_) => return crate::memory::MemoryEstimate::busy(),
5421 };
5422 if let Some(cache) = state.terminal_output_cache.as_ref() {
5423 terminal_output_caches = terminal_output_caches.saturating_add(1);
5424 bytes = bytes.saturating_add(terminal_output_cache_estimated_bytes(cache));
5425 }
5426 }
5427 let completion_count = match self.inner.completions.try_lock() {
5428 Ok(completions) => {
5429 for completion in completions.iter() {
5430 sessions.insert(completion.session_id.clone());
5431 bytes = bytes.saturating_add(completion_estimated_bytes(completion));
5432 }
5433 completions.len()
5434 }
5435 Err(_) => return crate::memory::MemoryEstimate::busy(),
5436 };
5437
5438 crate::memory::MemoryEstimate::estimated(bytes)
5439 .count("tasks", tasks.len())
5440 .count("sessions", sessions.len())
5441 .count("terminal_output_caches", terminal_output_caches)
5442 .count("completion_caches", completion_count)
5443 .count_u64("output_ring_bytes", 0)
5444 }
5445
5446 fn running_count(&self) -> usize {
5447 self.inner
5448 .tasks
5449 .lock()
5450 .map(|tasks| tasks.values().filter(|task| task.is_running()).count())
5451 .unwrap_or(0)
5452 }
5453
5454 fn start_watchdog(&self) {
5455 if !self.inner.watchdog_started.swap(true, Ordering::SeqCst) {
5456 super::watchdog::start(self.clone());
5457 }
5458 }
5459
5460 #[cfg(test)]
5461 pub fn task_json_path(&self, task_id: &str, session_id: &str) -> Option<PathBuf> {
5462 self.task_for_session(task_id, session_id)
5463 .map(|task| task.paths.json.clone())
5464 }
5465
5466 #[cfg(test)]
5467 pub fn task_exit_path(&self, task_id: &str, session_id: &str) -> Option<PathBuf> {
5468 self.task_for_session(task_id, session_id)
5469 .map(|task| task.paths.exit.clone())
5470 }
5471}
5472
5473#[cfg(unix)]
5474fn should_capture_pipeline_status(
5475 spawn_plan: &SpawnPlan,
5476 has_pipeline: bool,
5477 shell: &Path,
5478) -> bool {
5479 if spawn_plan.is_native_launcher() {
5480 return false;
5483 }
5484 has_pipeline && super::process::pipeline_shell_kind(shell).is_some()
5485}
5486
5487fn canonical_artifact_root(paths: &TaskPaths) -> PathBuf {
5488 fs::canonicalize(&paths.io_dir).unwrap_or_else(|_| paths.io_dir.clone())
5489}
5490
5491fn restart_fate_unknown_reason(metadata: &PersistedTask, paths: &TaskPaths) -> String {
5492 let output = match metadata.mode {
5493 BgMode::Pipes => &paths.stdout,
5494 BgMode::Pty => &paths.pty,
5495 };
5496 format!(
5497 "task {}: daemon restarted, process fate unknown, last output at {}",
5498 metadata.task_id,
5499 output.display()
5500 )
5501}
5502
5503fn append_pipeline_warning(
5508 cache: &mut TerminalOutputCache,
5509 metadata: &PersistedTask,
5510 paths: Option<&TaskPaths>,
5511) {
5512 if metadata.exit_code != Some(0) {
5513 return;
5514 }
5515 let Some(paths) = paths else {
5516 return;
5517 };
5518 if metadata.pipeline_segments.len() < 2 {
5519 return;
5520 }
5521 let mut status_file = match open_task_artifact(paths, TaskArtifact::PipelineStatus) {
5522 Ok(file) => file,
5523 Err(_) => {
5524 let Some(shell) = metadata.pipeline_status_unavailable.as_deref() else {
5525 return;
5526 };
5527 let footer = format!(
5528 "note: pipeline status unavailable under {shell}; an upstream failure may be masked by the final segment's exit code."
5529 );
5530 if cache.output_preview.trim().is_empty() {
5531 cache.output_preview = footer;
5532 } else {
5533 cache.output_preview = format!("{}\n{footer}", cache.output_preview.trim_end());
5534 }
5535 return;
5536 }
5537 };
5538 let Ok(status_bytes) = status_file.read_all() else {
5539 return;
5540 };
5541 let Some(statuses) = String::from_utf8_lossy(&status_bytes)
5542 .lines()
5543 .map(|line| line.trim().parse::<i32>().ok())
5544 .collect::<Option<Vec<_>>>()
5545 else {
5546 return;
5547 };
5548 if statuses.len() != metadata.pipeline_segments.len() {
5549 return;
5550 }
5551 let Some((failing_index, failing_code)) = statuses
5552 .iter()
5553 .enumerate()
5554 .take(statuses.len().saturating_sub(1))
5555 .find(|(_, code)| **code != 0)
5556 .map(|(index, code)| (index, *code))
5557 else {
5558 return;
5559 };
5560 let Some(final_segment) = metadata.pipeline_segments.last() else {
5561 return;
5562 };
5563 let failing_segment = &metadata.pipeline_segments[failing_index];
5564 let footer = format!(
5565 "note: `{}` (segment {} of {}) exited {}; the pipeline's exit code is `{}`'s.",
5566 failing_segment,
5567 failing_index + 1,
5568 metadata.pipeline_segments.len(),
5569 failing_code,
5570 final_segment,
5571 );
5572 if cache.output_preview.trim().is_empty() {
5573 cache.output_preview = footer;
5574 } else {
5575 cache.output_preview = format!("{}\n{}", cache.output_preview.trim_end(), footer,);
5576 }
5577}
5578
5579fn normalize_piped_display_output(text: &mut String) {
5582 if !text.contains('\r') {
5583 return;
5584 }
5585
5586 let mut rendered = String::with_capacity(text.len());
5587 let mut line = Vec::new();
5588 let mut column = 0;
5589 let mut chars = text.chars().peekable();
5590
5591 while let Some(character) = chars.next() {
5592 match character {
5593 '\r' if chars.peek() == Some(&'\n') => {
5594 chars.next();
5595 for character in &line {
5596 rendered.push(*character);
5597 }
5598 rendered.push('\n');
5599 line.clear();
5600 column = 0;
5601 }
5602 '\r' => column = 0,
5603 '\n' => {
5604 for character in &line {
5605 rendered.push(*character);
5606 }
5607 rendered.push('\n');
5608 line.clear();
5609 column = 0;
5610 }
5611 character => {
5612 if column < line.len() {
5613 line[column] = character;
5614 } else {
5615 line.resize(column, ' ');
5616 line.push(character);
5617 }
5618 column += 1;
5619 }
5620 }
5621 }
5622
5623 for character in &line {
5624 rendered.push(*character);
5625 }
5626 *text = rendered;
5627}
5628
5629fn render_compressed_with_recovery(
5630 buffer: &BgBuffer,
5631 mut compressed: CompressionResult,
5632 input_truncated: bool,
5633 disk_truncation: DiskTruncation,
5634 artifact_access: ArtifactRecoveryAccess,
5635) -> TerminalOutputCache {
5636 let compression_input_line_count = compressed.input_line_count;
5644 let had_trailing_newline = compressed.text.ends_with('\n');
5645 let mut text = strip_plain_truncation_marker_lines(&compressed.text)
5646 .trim_end()
5647 .to_string();
5648 if had_trailing_newline && !text.is_empty() {
5649 text.push('\n');
5650 }
5651 compressed.text = text;
5652
5653 let output_path = buffer.output_path().map(|path| path.display().to_string());
5654 let stderr_path = buffer.stderr_path().map(|path| path.display().to_string());
5655 let include_stderr_path = buffer.stream_len(StreamKind::Stderr) > 0;
5656 let mut recovery = RecoveryContext {
5657 dropped_by_class: compressed.dropped_by_class,
5658 had_inner_drop: compressed.had_inner_drop,
5659 offset_hint_eligible: compressed.offset_hint_eligible,
5660 offset_start_line: compressed.offset_start_line,
5661 byte_truncated: input_truncated,
5662 disk_truncated_prefix_bytes: disk_truncation.total_prefix_bytes(),
5663 output_path: output_path.clone(),
5664 stderr_path: stderr_path.clone(),
5665 include_stderr_path,
5666 artifact_access: artifact_access.clone(),
5667 };
5668
5669 let (output_preview, output_truncated) =
5670 render_body_with_recovery_marker(&compressed.text, &mut recovery);
5671 TerminalOutputCache {
5672 output_preview,
5673 output_truncated,
5674 compression_input_line_count: Some(compression_input_line_count),
5675 kind: TerminalOutputKind::Compressed,
5676 output_path,
5677 stderr_path,
5678 artifact_access,
5679 recovery: Some(recovery),
5680 }
5681}
5682
5683fn render_body_with_recovery_marker(body: &str, recovery: &mut RecoveryContext) -> (String, bool) {
5684 render_body_with_recovery_marker_at_cap(
5685 body,
5686 recovery,
5687 FINAL_OUTPUT_CAP_BYTES,
5688 cap_final_output,
5689 cap_final_output_with_marker,
5690 )
5691}
5692
5693fn render_raw_body_with_recovery_marker(
5694 body: &str,
5695 recovery: &mut RecoveryContext,
5696) -> (String, bool) {
5697 render_body_with_recovery_marker_at_cap(
5698 body,
5699 recovery,
5700 RAW_PASSTHROUGH_CAP_BYTES,
5701 |input| {
5702 super::output::cap_head_tail(
5703 input,
5704 RAW_PASSTHROUGH_CAP_BYTES,
5705 RAW_PASSTHROUGH_HEAD_BYTES,
5706 RAW_PASSTHROUGH_TAIL_BYTES,
5707 )
5708 },
5709 |input, marker| {
5710 super::output::cap_head_tail_with_marker(
5711 input,
5712 RAW_PASSTHROUGH_CAP_BYTES,
5713 RAW_PASSTHROUGH_HEAD_BYTES,
5714 RAW_PASSTHROUGH_TAIL_BYTES,
5715 marker,
5716 )
5717 },
5718 )
5719}
5720
5721fn render_body_with_recovery_marker_at_cap<F, G>(
5722 body: &str,
5723 recovery: &mut RecoveryContext,
5724 cap_bytes: usize,
5725 cap_plain: F,
5726 cap_with_marker: G,
5727) -> (String, bool)
5728where
5729 F: Fn(&str) -> super::output::CappedText,
5730 G: Fn(&str, &str) -> super::output::CappedText,
5731{
5732 let needs_marker = recovery.has_visible_drop();
5733 if body.len() > cap_bytes {
5734 recovery.byte_truncated = true;
5735 if let Some(marker) = recovery_marker(recovery) {
5736 let capped = cap_with_marker(body, &marker);
5737 return (capped.text, true);
5738 }
5739 let capped = cap_plain(body);
5740 return (capped.text, capped.truncated || needs_marker);
5741 }
5742
5743 if !needs_marker {
5744 return (body.to_string(), false);
5745 }
5746
5747 let Some(marker) = recovery_marker(recovery) else {
5748 return (body.to_string(), true);
5749 };
5750 let with_marker = append_recovery_marker(body, &marker);
5751 if with_marker.len() <= cap_bytes {
5752 return (with_marker, true);
5753 }
5754
5755 recovery.byte_truncated = true;
5756 let marker = recovery_marker(recovery).unwrap_or(marker);
5757 let capped = cap_with_marker(body, &marker);
5758 (capped.text, true)
5759}
5760
5761fn append_recovery_marker(body: &str, marker: &str) -> String {
5762 if body.is_empty() {
5763 return marker.to_string();
5764 }
5765 let mut output = body.trim_end().to_string();
5766 output.push('\n');
5767 output.push_str(marker);
5768 output
5769}
5770
5771fn recovery_marker(recovery: &RecoveryContext) -> Option<String> {
5772 let mut parts = Vec::new();
5773 for (class, count) in &recovery.dropped_by_class {
5774 let label = if *count == 1 {
5775 class.singular()
5776 } else {
5777 class.plural()
5778 };
5779 parts.push(format!("+{count} more {label}"));
5780 }
5781 if recovery.byte_truncated {
5782 parts.push("truncated output".to_string());
5783 }
5784 let disk_truncated_prefix_bytes = recovery.disk_truncated_prefix_bytes;
5785 if disk_truncated_prefix_bytes > 0 {
5786 parts.push(format!(
5787 "truncated {disk_truncated_prefix_bytes} bytes from saved output prefix"
5788 ));
5789 } else if recovery.had_inner_drop && parts.is_empty() {
5790 parts.push("omitted output".to_string());
5791 }
5792
5793 if parts.is_empty() {
5794 return None;
5795 }
5796
5797 let hint = recovery_hint(recovery);
5798 Some(format!("[{}; {hint}]", parts.join(", ")))
5799}
5800
5801fn bash_status_recovery_hint(access: &ArtifactRecoveryAccess) -> String {
5802 let task_id = serde_json::to_string(&access.task_id)
5803 .unwrap_or_else(|_| format!("\"{}\"", access.task_id));
5804 format!("use bash_status({{taskId: {task_id}}})")
5805}
5806
5807fn recovery_hint(recovery: &RecoveryContext) -> String {
5808 if !recovery.artifact_access.readable {
5809 return bash_status_recovery_hint(&recovery.artifact_access);
5810 }
5811
5812 if recovery.offset_hint_eligible
5816 && !recovery.byte_truncated
5817 && recovery.dropped_by_class.is_empty()
5818 && !recovery.include_stderr_path
5819 {
5820 if let (Some(path), Some(line)) =
5821 (recovery.output_path.as_deref(), recovery.offset_start_line)
5822 {
5823 return format!("see remaining: tail -n +{line} {}", quote_path(path));
5824 }
5825 }
5826
5827 let mut paths = Vec::new();
5828 if let Some(path) = recovery.output_path.as_deref() {
5829 paths.push(path);
5830 }
5831 if recovery.include_stderr_path {
5832 if let Some(path) = recovery.stderr_path.as_deref() {
5833 if !paths.contains(&path) {
5834 paths.push(path);
5835 }
5836 }
5837 }
5838
5839 if paths.is_empty() {
5840 return "full output unavailable".to_string();
5841 }
5842
5843 let reads = paths
5844 .into_iter()
5845 .map(|path| format!("read {}", quote_path(path)))
5846 .collect::<Vec<_>>()
5847 .join(" and ");
5848 if recovery.disk_truncated_prefix_bytes > 0 {
5849 format!("retained output: {reads}")
5850 } else {
5851 format!("full output: {reads}")
5852 }
5853}
5854
5855fn strip_plain_truncation_marker_lines(input: &str) -> String {
5856 input
5857 .lines()
5858 .filter(|line| !is_plain_truncation_marker(line.trim()))
5859 .collect::<Vec<_>>()
5860 .join("\n")
5861}
5862
5863fn strip_recovery_marker_lines(input: &str) -> String {
5864 input
5865 .lines()
5866 .filter(|line| !is_recovery_marker(line.trim()))
5867 .collect::<Vec<_>>()
5868 .join("\n")
5869}
5870
5871fn is_plain_truncation_marker(line: &str) -> bool {
5872 let Some(rest) = line.strip_prefix("...<truncated ") else {
5873 return false;
5874 };
5875 let Some(bytes) = rest.strip_suffix(" bytes>...") else {
5876 return false;
5877 };
5878 !bytes.is_empty() && bytes.chars().all(|ch| ch.is_ascii_digit())
5879}
5880
5881fn is_recovery_marker(line: &str) -> bool {
5882 line.starts_with('[')
5883 && line.ends_with(']')
5884 && (line.contains("full output: read ")
5885 || line.contains("retained output: read ")
5886 || line.contains("see remaining: tail -n +")
5887 || line.contains("use bash_status({taskId:")
5888 || line.contains("full output unavailable"))
5889}
5890
5891fn structured_output_pointer(
5892 total_bytes: u64,
5893 output_path: &str,
5894 truncated_prefix_bytes: u64,
5895 artifact_access: &ArtifactRecoveryAccess,
5896) -> String {
5897 if artifact_access.readable {
5898 return if truncated_prefix_bytes > 0 {
5899 retained_json_output_pointer(total_bytes, output_path, truncated_prefix_bytes)
5900 } else {
5901 json_output_pointer(total_bytes, output_path)
5902 };
5903 }
5904
5905 let kb = total_bytes.div_ceil(1024);
5906 let hint = bash_status_recovery_hint(artifact_access);
5907 if truncated_prefix_bytes > 0 {
5908 format!(
5909 "[JSON output {kb} KB; truncated {truncated_prefix_bytes} bytes from saved output prefix; retained output: {hint}]"
5910 )
5911 } else {
5912 format!("[JSON output {kb} KB; full output: {hint}]")
5913 }
5914}
5915
5916fn render_structured_output(
5917 command: &str,
5918 buffer: &BgBuffer,
5919 disk_truncation: DiskTruncation,
5920 artifact_access: ArtifactRecoveryAccess,
5921) -> Option<TerminalOutputCache> {
5922 if !is_gh_structured_command(command) {
5923 return None;
5924 }
5925
5926 let output_path = buffer
5927 .output_path()
5928 .map(|path| path.display().to_string())?;
5929 let stdout_bytes = buffer.stream_len(StreamKind::Stdout);
5930 if stdout_bytes == 0 {
5931 return None;
5932 }
5933
5934 if stdout_bytes > STRUCTURED_OUTPUT_CAP_BYTES as u64 {
5935 if !stream_starts_like_json(buffer, StreamKind::Stdout) {
5936 return None;
5937 }
5938 let output_preview = structured_output_pointer(
5939 stdout_bytes,
5940 &output_path,
5941 disk_truncation.total_prefix_bytes(),
5942 &artifact_access,
5943 );
5944 return Some(TerminalOutputCache {
5945 output_preview,
5946 output_truncated: true,
5947 compression_input_line_count: None,
5948 kind: TerminalOutputKind::Structured,
5949 output_path: Some(output_path),
5950 stderr_path: buffer.stderr_path().map(|path| path.display().to_string()),
5951 artifact_access,
5952 recovery: None,
5953 });
5954 }
5955
5956 let stdout = buffer.read_stream_bounded(StreamKind::Stdout, STRUCTURED_OUTPUT_CAP_BYTES);
5957 if stdout.truncated || !is_structured_body(&stdout.text) {
5958 return None;
5959 }
5960
5961 Some(TerminalOutputCache {
5962 output_preview: stdout.text,
5963 output_truncated: false,
5964 compression_input_line_count: None,
5965 kind: TerminalOutputKind::Structured,
5966 output_path: Some(output_path),
5967 stderr_path: buffer.stderr_path().map(|path| path.display().to_string()),
5968 artifact_access,
5969 recovery: None,
5970 })
5971}
5972
5973fn render_raw_passthrough(
5974 buffer: &BgBuffer,
5975 disk_truncation: DiskTruncation,
5976 artifact_access: ArtifactRecoveryAccess,
5977) -> TerminalOutputCache {
5978 let raw = buffer.read_combined_head_tail(
5979 RAW_PASSTHROUGH_CAP_BYTES,
5980 RAW_PASSTHROUGH_HEAD_BYTES,
5981 RAW_PASSTHROUGH_TAIL_BYTES,
5982 );
5983 let output_path = buffer.output_path().map(|path| path.display().to_string());
5984 let stderr_path = buffer.stderr_path().map(|path| path.display().to_string());
5985 if !raw.truncated && disk_truncation.total_prefix_bytes() == 0 {
5986 return TerminalOutputCache {
5987 output_preview: raw.text,
5988 output_truncated: false,
5989 compression_input_line_count: None,
5990 kind: TerminalOutputKind::Raw,
5991 output_path,
5992 stderr_path,
5993 artifact_access,
5994 recovery: None,
5995 };
5996 }
5997
5998 let include_stderr_path = buffer.stream_len(StreamKind::Stderr) > 0;
5999 let mut recovery = RecoveryContext {
6000 dropped_by_class: BTreeMap::new(),
6001 had_inner_drop: false,
6002 offset_hint_eligible: false,
6003 offset_start_line: None,
6004 byte_truncated: raw.truncated,
6005 disk_truncated_prefix_bytes: disk_truncation.total_prefix_bytes(),
6006 output_path: output_path.clone(),
6007 stderr_path: stderr_path.clone(),
6008 include_stderr_path,
6009 artifact_access: artifact_access.clone(),
6010 };
6011 let (output_preview, output_truncated) =
6012 render_raw_body_with_recovery_marker(&raw.text, &mut recovery);
6013 TerminalOutputCache {
6014 output_preview,
6015 output_truncated,
6016 compression_input_line_count: None,
6017 kind: TerminalOutputKind::Raw,
6018 output_path,
6019 stderr_path,
6020 artifact_access,
6021 recovery: Some(recovery),
6022 }
6023}
6024
6025fn append_bash_output_envelope(
6026 cache: &TerminalOutputCache,
6027 output: &mut String,
6028) -> Option<ListEnvelope> {
6029 cache
6030 .compression_input_line_count
6031 .and_then(|total| crate::list_surfaces::bash::append_envelope_trailer(output, total))
6032}
6033
6034fn completion_preview_for_cache(
6035 cache: &TerminalOutputCache,
6036 exit_code: Option<i32>,
6037) -> (String, bool) {
6038 let exit_ok = exit_code == Some(0);
6041 let threshold = completion_preview_threshold(exit_ok);
6042 if cache.kind == TerminalOutputKind::Structured && cache.output_preview.len() > threshold {
6043 if let Some(path) = cache.output_path.as_deref() {
6044 return (
6045 structured_output_pointer(
6046 cache.output_preview.len() as u64,
6047 path,
6048 0,
6049 &cache.artifact_access,
6050 ),
6051 true,
6052 );
6053 }
6054 return (cache.output_preview.clone(), cache.output_truncated);
6055 }
6056
6057 if let Some(recovery) = cache.recovery.as_ref() {
6058 if cache.output_preview.len() <= threshold {
6059 return (cache.output_preview.clone(), cache.output_truncated);
6060 }
6061 let body = strip_recovery_marker_lines(&cache.output_preview);
6062 let mut completion_recovery = recovery.clone();
6063 completion_recovery.byte_truncated = true;
6064 if let Some(marker) = recovery_marker(&completion_recovery) {
6065 let capped = cap_completion_output_with_marker(&body, &marker, exit_ok);
6066 return (capped.text, true);
6067 }
6068 }
6069
6070 let capped = cap_completion_output(&cache.output_preview, exit_ok);
6071 (capped.text, cache.output_truncated || capped.truncated)
6072}
6073
6074fn is_gh_structured_command(command: &str) -> bool {
6075 let Some(normalized) = crate::compress::plain_command_for_structured_output(command) else {
6076 return false;
6077 };
6078 let tokens = shell_words_for_flags(&normalized);
6079 let Some(head) = tokens.first() else {
6080 return false;
6081 };
6082 let head_name = Path::new(head)
6083 .file_name()
6084 .and_then(|name| name.to_str())
6085 .unwrap_or(head);
6086 if !(head_name == "gh" || head_name.eq_ignore_ascii_case("gh.exe")) {
6087 return false;
6088 }
6089 tokens.iter().any(|token| {
6090 matches!(token.as_str(), "--json" | "--jq" | "--template")
6091 || token.starts_with("--json=")
6092 || token.starts_with("--jq=")
6093 || token.starts_with("--template=")
6094 })
6095}
6096
6097fn shell_words_for_flags(command: &str) -> Vec<String> {
6098 let mut words = Vec::new();
6099 let mut current = String::new();
6100 let mut in_single = false;
6101 let mut in_double = false;
6102 let mut escaped = false;
6103
6104 for ch in command.chars() {
6105 if escaped {
6106 current.push(ch);
6107 escaped = false;
6108 continue;
6109 }
6110 if ch == '\\' && !in_single {
6111 escaped = true;
6112 continue;
6113 }
6114 if ch == '\'' && !in_double {
6115 in_single = !in_single;
6116 continue;
6117 }
6118 if ch == '"' && !in_single {
6119 in_double = !in_double;
6120 continue;
6121 }
6122 if ch.is_whitespace() && !in_single && !in_double {
6123 if !current.is_empty() {
6124 words.push(std::mem::take(&mut current));
6125 }
6126 continue;
6127 }
6128 if matches!(ch, ';' | '&' | '|') && !in_single && !in_double {
6129 if !current.is_empty() {
6130 words.push(std::mem::take(&mut current));
6131 }
6132 continue;
6133 }
6134 current.push(ch);
6135 }
6136 if !current.is_empty() {
6137 words.push(current);
6138 }
6139 words
6140}
6141
6142fn is_structured_body(body: &str) -> bool {
6143 let trimmed = body.trim();
6144 if trimmed.is_empty() {
6145 return false;
6146 }
6147 if serde_json::from_str::<serde_json::Value>(trimmed).is_ok() {
6148 return true;
6149 }
6150
6151 let mut saw_line = false;
6152 for line in trimmed
6153 .lines()
6154 .map(str::trim)
6155 .filter(|line| !line.is_empty())
6156 {
6157 saw_line = true;
6158 if serde_json::from_str::<serde_json::Value>(line).is_err() {
6159 return false;
6160 }
6161 }
6162 saw_line
6163}
6164
6165fn stream_starts_like_json(buffer: &BgBuffer, stream: StreamKind) -> bool {
6166 buffer
6167 .read_stream_bounded(stream, 512)
6168 .text
6169 .chars()
6170 .find(|ch| !ch.is_whitespace())
6171 .is_some_and(|ch| matches!(ch, '{' | '[' | '"' | '-' | '0'..='9' | 't' | 'f' | 'n'))
6172}
6173
6174struct CompletionTokenCounts {
6175 original_tokens: Option<u32>,
6176 compressed_tokens: Option<u32>,
6177 original_bytes: Option<i64>,
6178 compressed_bytes: Option<i64>,
6179 tokens_skipped: bool,
6180}
6181
6182impl CompletionTokenCounts {
6183 fn skipped() -> Self {
6184 Self {
6185 original_tokens: None,
6186 compressed_tokens: None,
6187 original_bytes: None,
6188 compressed_bytes: None,
6189 tokens_skipped: true,
6190 }
6191 }
6192}
6193
6194fn live_descendants_summary(metadata: &PersistedTask) -> Option<String> {
6195 let Some(members) = metadata.live_descendants.as_deref() else {
6196 #[cfg(windows)]
6197 return (metadata.mode == BgMode::Pipes).then(|| {
6198 "live descendant check n/a on Windows (background tasks do not use a Job Object)"
6199 .to_string()
6200 });
6201 #[cfg(not(windows))]
6202 return None;
6203 };
6204 let total = members.len() + metadata.live_descendants_omitted;
6205 if total == 0 {
6206 return None;
6207 }
6208
6209 let mut counts = BTreeMap::<&str, usize>::new();
6210 for member in members {
6211 *counts.entry(member.comm.as_str()).or_default() += 1;
6212 }
6213 let mut processes = counts
6214 .into_iter()
6215 .map(|(comm, count)| {
6216 if count == 1 {
6217 comm.to_string()
6218 } else {
6219 format!("{comm} ×{count}")
6220 }
6221 })
6222 .collect::<Vec<_>>()
6223 .join(", ");
6224 if metadata.live_descendants_omitted > 0 {
6225 processes.push_str(&format!(" +{} more", metadata.live_descendants_omitted));
6226 }
6227 let workload = ["vitest", "jest", "cargo test"]
6228 .into_iter()
6229 .find(|name| metadata.command.contains(name))
6230 .map(|name| format!(" ({name})"))
6231 .unwrap_or_default();
6232 Some(format!(
6233 "{total} live descendants still running: {processes}{workload} — they keep the task's process group; bash_kill({}) stops them",
6234 metadata.task_id
6235 ))
6236}
6237
6238fn completion_status_text(status: &BgTaskStatus, exit_code: Option<i32>) -> String {
6239 match status {
6240 BgTaskStatus::TimedOut => "timed out".to_string(),
6241 BgTaskStatus::Killed => "killed".to_string(),
6242 _ => exit_code
6243 .map(|code| format!("exit {code}"))
6244 .unwrap_or_else(|| format!("{status:?}").to_lowercase()),
6245 }
6246}
6247
6248fn token_count_u32(text: &str) -> u32 {
6249 aft_tokenizer::count_tokens(text)
6250 .try_into()
6251 .unwrap_or(u32::MAX)
6252}
6253
6254impl Default for BgTaskRegistry {
6255 fn default() -> Self {
6256 Self::new(Arc::new(Mutex::new(None)))
6257 }
6258}
6259
6260fn session_recovery_key(storage_dir: &Path, session_id: &str) -> Option<String> {
6263 session_tasks_dir(storage_dir, session_id)
6264 .file_name()
6265 .and_then(|name| name.to_str())
6266 .map(str::to_string)
6267}
6268
6269fn concurrently_replaced(error: &std::io::Error) -> bool {
6274 error.kind() == std::io::ErrorKind::Interrupted
6275 && error
6276 .to_string()
6277 .contains(super::persistence::ARTIFACT_CONCURRENTLY_REPLACED)
6278}
6279
6280fn modified_within(path: &Path, grace: Duration) -> bool {
6281 fs::metadata(path)
6282 .and_then(|metadata| metadata.modified())
6283 .ok()
6284 .and_then(|modified| SystemTime::now().duration_since(modified).ok())
6285 .map(|age| age < grace)
6286 .unwrap_or(false)
6287}
6288
6289fn canonicalized_path(path: &Path) -> PathBuf {
6290 fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf())
6291}
6292
6293fn started_instant_from_unix_millis(started_at: u64) -> Instant {
6294 let now_ms = SystemTime::now()
6295 .duration_since(UNIX_EPOCH)
6296 .ok()
6297 .map(|duration| duration.as_millis() as u64)
6298 .unwrap_or(started_at);
6299 let elapsed_ms = now_ms.saturating_sub(started_at);
6300 Instant::now()
6301 .checked_sub(Duration::from_millis(elapsed_ms))
6302 .unwrap_or_else(Instant::now)
6303}
6304
6305fn gc_quarantine(storage_dir: &Path) {
6306 let quarantine_root = storage_dir.join("bash-tasks-quarantine");
6307 let Ok(session_dirs) = fs::read_dir(&quarantine_root) else {
6308 return;
6309 };
6310 for session_entry in session_dirs.flatten() {
6311 let session_quarantine_dir = session_entry.path();
6312 if !session_quarantine_dir.is_dir() {
6313 continue;
6314 }
6315 let entries = match fs::read_dir(&session_quarantine_dir) {
6316 Ok(entries) => entries,
6317 Err(error) => {
6318 crate::slog_warn!(
6319 "failed to read background task quarantine dir {}: {error}",
6320 session_quarantine_dir.display()
6321 );
6322 continue;
6323 }
6324 };
6325 for entry in entries.flatten() {
6326 let path = entry.path();
6327 if modified_within(&path, QUARANTINE_GC_GRACE) {
6328 continue;
6329 }
6330 let result = if path.is_dir() {
6331 fs::remove_dir_all(&path)
6332 } else {
6333 fs::remove_file(&path)
6334 };
6335 match result {
6336 Ok(()) => log::debug!(
6337 "deleted old background task quarantine entry {}",
6338 path.display()
6339 ),
6340 Err(error) => crate::slog_warn!(
6341 "failed to delete old background task quarantine entry {}: {error}",
6342 path.display()
6343 ),
6344 }
6345 }
6346 let _ = fs::remove_dir(&session_quarantine_dir);
6347 }
6348 let _ = fs::remove_dir(&quarantine_root);
6349}
6350
6351fn read_for_token_count_from_disk(
6352 metadata: &PersistedTask,
6353 paths: &TaskPaths,
6354 max_bytes_per_stream: usize,
6355) -> TokenCountInput {
6356 if metadata.mode == BgMode::Pty {
6357 return TokenCountInput::Skipped;
6358 }
6359 let stdout = read_file_tail_capped(paths, TaskArtifact::Stdout, max_bytes_per_stream);
6366 let stderr = read_file_tail_capped(paths, TaskArtifact::Stderr, max_bytes_per_stream);
6367 match (stdout, stderr) {
6368 (Ok(stdout), Ok(stderr)) => TokenCountInput::Text(combine_streams(
6369 String::from_utf8_lossy(&stdout).as_ref(),
6370 String::from_utf8_lossy(&stderr).as_ref(),
6371 )),
6372 (Ok(stdout), Err(_)) => TokenCountInput::Text(combine_streams(
6373 String::from_utf8_lossy(&stdout).as_ref(),
6374 "",
6375 )),
6376 (Err(_), Ok(stderr)) => TokenCountInput::Text(combine_streams(
6377 "",
6378 String::from_utf8_lossy(&stderr).as_ref(),
6379 )),
6380 (Err(_), Err(_)) => TokenCountInput::Skipped,
6381 }
6382}
6383
6384fn read_file_tail_capped(
6385 paths: &TaskPaths,
6386 artifact: TaskArtifact,
6387 max_bytes: usize,
6388) -> std::io::Result<Vec<u8>> {
6389 let mut file = open_task_artifact(paths, artifact)?;
6390 file.tail(max_bytes).map(|(bytes, _)| bytes)
6391}
6392
6393fn task_bundle_is_absent(storage_dir: &Path, session_id: &str, task_id: &str) -> bool {
6394 let session_dir = session_tasks_dir(storage_dir, session_id);
6395 !session_dir.join(task_id).exists() && !session_dir.join(format!("{task_id}.json")).exists()
6396}
6397
6398fn terminal_db_row_snapshot(row: BashTaskRow, metadata: PersistedTask) -> BgTaskSnapshot {
6399 let existing_path = |path: Option<String>| {
6400 path.filter(|path| {
6401 fs::metadata(path)
6402 .map(|metadata| metadata.is_file())
6403 .unwrap_or(false)
6404 })
6405 };
6406 let duration_ms = metadata.duration_ms.or_else(|| {
6407 metadata
6408 .finished_at
6409 .map(|finished_at| finished_at.saturating_sub(metadata.started_at))
6410 });
6411 let live_descendants_summary = live_descendants_summary(&metadata);
6412 BgTaskSnapshot {
6413 info: BgTaskInfo {
6414 task_id: metadata.task_id,
6415 status: metadata.status,
6416 command: metadata.command,
6417 mode: metadata.mode.clone(),
6418 started_at: metadata.started_at,
6419 duration_ms,
6420 status_reason: metadata.status_reason,
6421 },
6422 exit_code: metadata.exit_code,
6423 child_pid: metadata.child_pid,
6424 workdir: metadata.workdir.display().to_string(),
6425 output_preview: String::new(),
6426 bash_output_list_envelope: None,
6427 output_truncated: false,
6428 output_path: existing_path(row.stdout_path),
6429 stderr_path: existing_path(row.stderr_path),
6430 pty_rows: (metadata.mode == BgMode::Pty).then_some(metadata.pty_rows.unwrap_or(24)),
6431 pty_cols: (metadata.mode == BgMode::Pty).then_some(metadata.pty_cols.unwrap_or(80)),
6432 pty_screen: None,
6433 scanner_report: metadata.scanner_report,
6434 sandbox_native: metadata.sandbox_native,
6435 sandbox_unavailable: false,
6436 live_descendants: metadata.live_descendants.clone(),
6437 live_descendants_omitted: metadata.live_descendants_omitted,
6438 live_descendants_summary,
6439 kill_signaled: false,
6440 kill_reached: 0,
6441 }
6442}
6443
6444impl BgTask {
6445 fn snapshot(&self, preview_bytes: usize) -> BgTaskSnapshot {
6446 let state = self
6447 .state
6448 .lock()
6449 .unwrap_or_else(|poison| poison.into_inner());
6450 self.snapshot_locked(&state, preview_bytes)
6451 }
6452
6453 fn snapshot_locked(&self, state: &BgTaskState, preview_bytes: usize) -> BgTaskSnapshot {
6454 let metadata = &state.metadata;
6455 let duration_ms = metadata.duration_ms.or_else(|| {
6456 metadata
6457 .status
6458 .is_terminal()
6459 .then(|| self.started.elapsed().as_millis() as u64)
6460 });
6461 let (output_preview, output_truncated) = if metadata.mode == BgMode::Pty {
6462 (String::new(), false)
6463 } else if metadata.status.is_terminal() {
6464 state
6465 .terminal_output_cache
6466 .as_ref()
6467 .map(|cache| (cache.output_preview.clone(), cache.output_truncated))
6468 .unwrap_or_else(|| (String::new(), false))
6469 } else if preview_bytes == 0 {
6470 (String::new(), false)
6471 } else {
6472 state.buffer.read_tail(preview_bytes)
6473 };
6474 BgTaskSnapshot {
6475 info: BgTaskInfo {
6476 task_id: self.task_id.clone(),
6477 status: metadata.status.clone(),
6478 command: metadata.command.clone(),
6479 mode: metadata.mode.clone(),
6480 started_at: metadata.started_at,
6481 duration_ms,
6482 status_reason: metadata.status_reason.clone(),
6483 },
6484 exit_code: metadata.exit_code,
6485 child_pid: metadata.child_pid,
6486 workdir: metadata.workdir.display().to_string(),
6487 output_preview,
6488 bash_output_list_envelope: None,
6489 output_truncated,
6490 output_path: state
6491 .buffer
6492 .output_path()
6493 .map(|path| path.display().to_string()),
6494 stderr_path: state
6495 .buffer
6496 .stderr_path()
6497 .map(|path| path.display().to_string()),
6498 pty_rows: (metadata.mode == BgMode::Pty).then_some(metadata.pty_rows.unwrap_or(24)),
6499 pty_cols: (metadata.mode == BgMode::Pty).then_some(metadata.pty_cols.unwrap_or(80)),
6500 pty_screen: None,
6501 scanner_report: metadata.scanner_report.clone(),
6502 sandbox_native: metadata.sandbox_native,
6503 sandbox_unavailable: metadata.sandbox_native
6504 && open_task_artifact(&self.paths, TaskArtifact::SandboxUnavailable)
6505 .and_then(|mut file| file.read_all())
6506 .is_ok_and(|bytes| bytes == b"sandbox_unavailable"),
6507 live_descendants: metadata.live_descendants.clone(),
6508 live_descendants_omitted: metadata.live_descendants_omitted,
6509 live_descendants_summary: live_descendants_summary(metadata),
6510 kill_signaled: false,
6511 kill_reached: 0,
6512 }
6513 }
6514
6515 pub(crate) fn is_running(&self) -> bool {
6516 self.state
6517 .lock()
6518 .map(|state| {
6519 state.metadata.status == BgTaskStatus::Running
6520 || (state.metadata.mode == BgMode::Pty
6521 && state.metadata.status == BgTaskStatus::Killing)
6522 })
6523 .unwrap_or(false)
6524 }
6525
6526 fn is_terminal(&self) -> bool {
6527 self.state
6528 .lock()
6529 .map(|state| state.metadata.status.is_terminal())
6530 .unwrap_or(false)
6531 }
6532
6533 fn mark_terminal_now(&self) {
6534 if let Ok(mut terminal_at) = self.terminal_at.lock() {
6535 if terminal_at.is_none() {
6536 *terminal_at = Some(Instant::now());
6537 }
6538 }
6539 }
6540
6541 fn set_completion_delivered(
6542 &self,
6543 delivered: bool,
6544 registry: &BgTaskRegistry,
6545 ) -> std::io::Result<()> {
6546 let mut state = self
6547 .state
6548 .lock()
6549 .map_err(|_| std::io::Error::other("background task lock poisoned"))?;
6550 let updated = registry.update_task_metadata(&self.paths, |metadata| {
6551 metadata.completion_delivered = delivered;
6552 })?;
6553 state.metadata = updated;
6554 Ok(())
6555 }
6556}
6557
6558#[cfg(unix)]
6579fn reap_piped_child(child_slot: &mut Option<Child>) {
6580 if let Some(mut child) = child_slot.take() {
6581 if matches!(child.try_wait(), Ok(None)) {
6582 let _ = child.wait();
6583 }
6584 }
6585}
6586
6587#[cfg(windows)]
6592fn reap_piped_child(child_slot: &mut Option<Child>) {
6593 *child_slot = None;
6594}
6595
6596fn terminal_metadata_from_marker(
6597 mut metadata: PersistedTask,
6598 marker: ExitMarker,
6599 reason: Option<String>,
6600) -> PersistedTask {
6601 match marker {
6602 ExitMarker::Code(code) => {
6603 let status = if code == 0 {
6604 BgTaskStatus::Completed
6605 } else {
6606 BgTaskStatus::Failed
6607 };
6608 metadata.mark_terminal(status, Some(code), reason);
6609 }
6610 ExitMarker::Killed => metadata.mark_terminal(
6611 BgTaskStatus::Killed,
6612 terminal_exit_code_for_status(&BgTaskStatus::Killed),
6613 reason,
6614 ),
6615 }
6616 metadata
6617}
6618
6619fn terminal_exit_code_for_status(status: &BgTaskStatus) -> Option<i32> {
6620 match status {
6621 BgTaskStatus::TimedOut => Some(124),
6622 BgTaskStatus::Killed => Some(137),
6623 _ => None,
6624 }
6625}
6626
6627fn attach_sandbox_metadata(metadata: &mut PersistedTask, spawn_plan: &SpawnPlan) {
6628 metadata.sandbox_native = spawn_plan.is_native_launcher();
6629 metadata.sandbox_temp_dir = spawn_plan.temp_dir().map(Path::to_path_buf);
6630}
6631
6632#[cfg(unix)]
6633pub(crate) fn resolve_posix_shell() -> PathBuf {
6634 static POSIX_SHELL: OnceLock<PathBuf> = OnceLock::new();
6635 POSIX_SHELL
6636 .get_or_init(|| {
6637 std::env::var_os("BASH")
6638 .filter(|value| !value.is_empty())
6639 .map(PathBuf::from)
6640 .filter(|path| path.exists())
6641 .or_else(|| which::which("bash").ok())
6642 .or_else(|| which::which("zsh").ok())
6643 .unwrap_or_else(|| PathBuf::from("/bin/sh"))
6644 })
6645 .clone()
6646}
6647
6648#[cfg(windows)]
6649fn detached_shell_command_for(
6650 shell: crate::windows_shell::WindowsShell,
6651 command: &str,
6652 exit_path: &Path,
6653 paths: &TaskPaths,
6654 creation_flags: u32,
6655) -> Result<Command, String> {
6656 use crate::windows_shell::WindowsShell;
6657 let wrapper_body = shell.wrapper_script_bytes(command, exit_path);
6670 let wrapper_ext = match shell {
6671 WindowsShell::Pwsh | WindowsShell::Powershell => "ps1",
6672 WindowsShell::Cmd => "bat",
6673 WindowsShell::Posix(_) => "sh",
6677 };
6678 let wrapper_path = paths.dir.join(format!(
6679 "{}.{}",
6680 paths
6681 .json
6682 .file_stem()
6683 .and_then(|s| s.to_str())
6684 .unwrap_or("wrapper"),
6685 wrapper_ext
6686 ));
6687 fs::write(&wrapper_path, wrapper_body)
6688 .map_err(|e| format!("failed to write background bash wrapper script: {e}"))?;
6689
6690 let mut cmd = Command::new(shell.binary().as_ref());
6691 match shell {
6692 WindowsShell::Pwsh | WindowsShell::Powershell => {
6693 cmd.args([
6696 "-NoLogo",
6697 "-NoProfile",
6698 "-NonInteractive",
6699 "-ExecutionPolicy",
6700 "Bypass",
6701 "-File",
6702 ]);
6703 cmd.arg(&wrapper_path);
6704 }
6705 WindowsShell::Cmd => {
6706 cmd.args(["/D", "/C"]);
6713 cmd.arg(&wrapper_path);
6714 }
6715 WindowsShell::Posix(_) => {
6716 cmd.arg(&wrapper_path);
6721 }
6722 }
6723
6724 cmd.creation_flags(creation_flags);
6728 Ok(cmd)
6729}
6730
6731fn spawn_detached_child(
6747 spawn_plan: &SpawnPlan,
6748 command: &str,
6749 shell: super::BashShell,
6750 shell_path: &Path,
6751 paths: &TaskPaths,
6752 workdir: &Path,
6753 env: &HashMap<String, String>,
6754 io_handles: &mut TaskIoHandles,
6755 capture_pipeline_status: bool,
6756 #[cfg_attr(not(target_os = "linux"), allow(unused_variables))] linux_scope: bool,
6757) -> Result<std::process::Child, String> {
6758 #[cfg(windows)]
6759 let _ = capture_pipeline_status;
6760 #[cfg(not(windows))]
6761 let _ = (command, shell);
6762 #[cfg(not(windows))]
6763 {
6764 use std::os::fd::AsRawFd;
6765
6766 let stdout = io_handles
6767 .clone_file(TaskArtifact::Stdout)
6768 .map_err(|e| format!("failed to clone stdout capture handle: {e}"))?;
6769 let stderr = io_handles
6770 .clone_file(TaskArtifact::Stderr)
6771 .map_err(|e| format!("failed to clone stderr capture handle: {e}"))?;
6772 let prepared = spawn_plan
6773 .prepared_task()
6774 .ok_or_else(|| "background task payload was not prepared".to_string())?;
6775 let payload = prepared.invocation()?;
6776 let exit = io_handles
6777 .inheritable_file(TaskArtifact::Exit)
6778 .map_err(|e| format!("failed to inherit exit marker handle: {e}"))?;
6779 let failure = io_handles
6780 .inheritable_file(TaskArtifact::SandboxUnavailable)
6781 .map_err(|e| format!("failed to inherit sandbox failure marker handle: {e}"))?;
6782 let pipeline_status = capture_pipeline_status
6783 .then(|| io_handles.inheritable_file(TaskArtifact::PipelineStatus))
6784 .transpose()
6785 .map_err(|e| format!("failed to inherit pipeline status handle: {e}"))?;
6786 let shell_path = spawn_plan.host_shell_path().unwrap_or(shell_path);
6787 let pipeline_shell = super::process::pipeline_shell_kind(shell_path).unwrap_or("");
6788 let pipeline_status_fd = if capture_pipeline_status {
6789 crate::sandbox_spawn::CHILD_PIPE_STATUS_FD.to_string()
6790 } else {
6791 String::new()
6792 };
6793 let base_executable = PathBuf::from("/bin/sh");
6794 let base_args = vec![
6795 OsString::from("-c"),
6796 payload.wrapper_text.clone(),
6797 OsString::from("aft-payload-wrapper"),
6798 shell_path.as_os_str().to_os_string(),
6799 payload.command_text.clone(),
6800 OsString::from(crate::sandbox_spawn::CHILD_EXIT_FD.to_string()),
6801 OsString::from(pipeline_status_fd),
6802 OsString::from(pipeline_shell),
6803 ];
6804 #[cfg(target_os = "linux")]
6805 let (executable, args) = if linux_scope {
6806 if spawn_plan.is_native_launcher() {
6807 crate::slog_info!(
6808 "bash.linux_scope requested but native sandbox launch cannot reach the user manager; spawning without a systemd scope"
6809 );
6810 (base_executable, base_args)
6811 } else if let Some(launcher) = super::process::select_systemd_scope_launcher() {
6812 super::process::systemd_scope_argv(&launcher, &base_executable, &base_args)
6813 } else {
6814 crate::slog_info!(
6815 "bash.linux_scope requested but systemd-run or the user manager is unavailable; spawning without a systemd scope"
6816 );
6817 (base_executable, base_args)
6818 }
6819 } else {
6820 (base_executable, base_args)
6821 };
6822 #[cfg(not(target_os = "linux"))]
6823 let (executable, args) = {
6824 let _ = linux_scope;
6825 (base_executable, base_args)
6826 };
6827 let (mut child_command, profile_handle) = crate::sandbox_spawn::detached_command_for_plan(
6828 spawn_plan,
6829 executable.as_os_str(),
6830 &args,
6831 &paths.json,
6832 crate::sandbox_spawn::CHILD_EXIT_FD,
6833 crate::sandbox_spawn::CHILD_FAILURE_FD,
6834 )?;
6835 crate::sandbox_spawn::apply_marker_fd_allowlist(
6836 &mut child_command,
6837 exit.as_raw_fd(),
6838 failure.as_raw_fd(),
6839 pipeline_status.as_ref().map(|file| file.as_raw_fd()),
6840 )?;
6841 child_command
6842 .current_dir(workdir)
6843 .stdin(Stdio::null())
6844 .stdout(Stdio::from(stdout))
6845 .stderr(Stdio::from(stderr));
6846 crate::agent_child_env::apply_to_command(&mut child_command, env);
6847 crate::sandbox_spawn::apply_sandbox_environment(spawn_plan, &mut child_command, env);
6848 let child = child_command
6849 .spawn()
6850 .map_err(|e| format!("failed to spawn background bash command: {e}"));
6851 drop((payload, exit, failure, pipeline_status, profile_handle));
6852 child
6853 }
6854 #[cfg(windows)]
6855 {
6856 let _ = shell_path;
6857 use crate::windows_shell::shell_candidates;
6858 match spawn_plan {
6859 SpawnPlan::Unsandboxed | SpawnPlan::Host { .. } => {}
6860 SpawnPlan::Refused { code, .. } => return Err((*code).to_string()),
6861 SpawnPlan::Launcher { .. } => return Err("sandbox_unavailable".to_string()),
6862 }
6863 let candidates: Vec<crate::windows_shell::WindowsShell> = if shell.is_powershell() {
6874 vec![crate::windows_shell::WindowsShell::Pwsh]
6875 } else {
6876 shell_candidates()
6877 };
6878 const FLAG_CREATE_NEW_PROCESS_GROUP: u32 = 0x0000_0200;
6891 const FLAG_CREATE_BREAKAWAY_FROM_JOB: u32 = 0x0100_0000;
6892 const FLAG_CREATE_NO_WINDOW: u32 = 0x0800_0000;
6893 let with_breakaway =
6894 FLAG_CREATE_NO_WINDOW | FLAG_CREATE_NEW_PROCESS_GROUP | FLAG_CREATE_BREAKAWAY_FROM_JOB;
6895 let without_breakaway = FLAG_CREATE_NO_WINDOW | FLAG_CREATE_NEW_PROCESS_GROUP;
6896 let mut last_error: Option<String> = None;
6897 for (idx, shell) in candidates.iter().enumerate() {
6898 for &flags in &[with_breakaway, without_breakaway] {
6902 let stdout = io_handles
6905 .clone_file(TaskArtifact::Stdout)
6906 .map_err(|e| format!("failed to clone stdout capture handle: {e}"))?;
6907 let stderr = io_handles
6908 .clone_file(TaskArtifact::Stderr)
6909 .map_err(|e| format!("failed to clone stderr capture handle: {e}"))?;
6910 let mut cmd =
6911 detached_shell_command_for(shell.clone(), command, &paths.exit, paths, flags)?;
6912 cmd.current_dir(workdir)
6913 .stdin(Stdio::null())
6914 .stdout(Stdio::from(stdout))
6915 .stderr(Stdio::from(stderr));
6916 crate::agent_child_env::apply_to_command(&mut cmd, env);
6917 match cmd.spawn() {
6918 Ok(child) => {
6919 if idx > 0 {
6920 crate::slog_warn!("background bash spawn fell back to {} after {} earlier candidate(s) failed; \
6921 the cached PATH probe disagreed with runtime spawn — likely PATH \
6922 inheritance, antivirus / AppLocker / Defender ASR, or sandbox policy.",
6923 shell.binary(),
6924 idx);
6925 }
6926 if flags == without_breakaway {
6927 crate::slog_warn!(
6928 "background bash spawn: CREATE_BREAKAWAY_FROM_JOB rejected \
6929 (likely a restrictive Job Object — CI sandbox or MDM policy). \
6930 Spawned without breakaway; the bg task will be torn down if the \
6931 AFT process group is killed."
6932 );
6933 }
6934 return Ok(child);
6935 }
6936 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
6937 crate::slog_warn!("background bash spawn: {} returned NotFound at runtime — trying next candidate",
6938 shell.binary());
6939 last_error = Some(format!("{}: {e}", shell.binary()));
6940 break;
6943 }
6944 Err(e) if flags == with_breakaway && e.raw_os_error() == Some(5) => {
6945 crate::slog_warn!(
6947 "background bash spawn: CREATE_BREAKAWAY_FROM_JOB rejected with \
6948 Access Denied — retrying {} without breakaway",
6949 shell.binary()
6950 );
6951 last_error = Some(format!("{}: {e}", shell.binary()));
6952 continue;
6953 }
6954 Err(e) => {
6955 return Err(format!(
6956 "failed to spawn background bash command via {}: {e}",
6957 shell.binary()
6958 ));
6959 }
6960 }
6961 }
6962 }
6963 Err(format!(
6964 "failed to spawn background bash command: no Windows shell could be spawned. \
6965 Last error: {}. PATH-probed candidates: {:?}",
6966 last_error.unwrap_or_else(|| "no candidates were attempted".to_string()),
6967 candidates.iter().map(|s| s.binary()).collect::<Vec<_>>()
6968 ))
6969 }
6970}
6971
6972#[cfg(test)]
6973fn random_slug() -> String {
6974 let mut bytes = [0u8; 8];
6982 getrandom::fill(&mut bytes).unwrap_or_else(|_| {
6984 let t = SystemTime::now()
6986 .duration_since(UNIX_EPOCH)
6987 .map(|d| d.as_nanos() as u64)
6988 .unwrap_or(0);
6989 let p = u64::from(std::process::id());
6990 bytes.copy_from_slice(&(t ^ p.rotate_left(32)).to_le_bytes());
6991 });
6992 let hex: String = bytes.iter().map(|b| format!("{b:02x}")).collect();
6994 format!("bash-{hex}")
6995}
6996
6997#[cfg(test)]
6998mod tests {
6999 use std::collections::HashMap;
7000 use std::fs;
7001 use std::io::Write;
7002 #[cfg(unix)]
7003 use std::os::unix::fs::PermissionsExt;
7004 use std::sync::atomic::{AtomicBool, AtomicUsize};
7005 use std::sync::{Arc, Mutex};
7006 use std::time::{Duration, Instant, SystemTime};
7007
7008 use super::*;
7009 use crate::bash_background::persistence::{read_task, task_paths, write_task};
7010
7011 #[cfg(unix)]
7012 const QUICK_SUCCESS_COMMAND: &str = "true";
7013 #[cfg(windows)]
7014 const QUICK_SUCCESS_COMMAND: &str = "cmd /c exit 0";
7015
7016 const CHILD_EXIT_LIVENESS_BOUND: Duration = Duration::from_secs(30);
7021
7022 #[cfg(unix)]
7023 const LONG_RUNNING_COMMAND: &str = "sleep 5";
7024
7025 #[cfg(unix)]
7026 #[test]
7027 fn launcher_plans_disable_pipeline_status_capture() {
7028 let launcher = SpawnPlan::launcher_for_test(
7029 crate::sandbox_profile::SandboxProfile {
7030 v: crate::sandbox_profile::SANDBOX_PROFILE_VERSION,
7031 writable_roots: Vec::new(),
7032 write_deny: Vec::new(),
7033 write_deny_nested: Vec::new(),
7034 read_allow: Vec::new(),
7035 read_deny: Vec::new(),
7036 socket_deny: Vec::new(),
7037 cache_roots: Vec::new(),
7038 temp_dir: PathBuf::from("/tmp/aft-test-sandbox"),
7039 },
7040 PathBuf::from("/bin/true"),
7041 );
7042 assert!(!should_capture_pipeline_status(
7043 &launcher,
7044 true,
7045 Path::new("/bin/bash")
7046 ));
7047 assert!(should_capture_pipeline_status(
7048 &SpawnPlan::Unsandboxed,
7049 true,
7050 Path::new("/bin/bash")
7051 ));
7052 }
7053
7054 #[cfg(windows)]
7055 const LONG_RUNNING_COMMAND: &str = "cmd /c timeout /t 5 /nobreak > nul";
7056
7057 #[test]
7058 fn bash_memory_estimate_is_zero_when_empty_and_nonzero_for_completion_cache() {
7059 let registry = BgTaskRegistry::default();
7060 assert_eq!(registry.estimated_memory().estimated_bytes, Some(0));
7061 registry
7062 .inner
7063 .completions
7064 .lock()
7065 .unwrap()
7066 .push_back(BgCompletion {
7067 task_id: "bash-memory".to_string(),
7068 session_id: "session-memory".to_string(),
7069 status: BgTaskStatus::Completed,
7070 exit_code: Some(0),
7071 command: "printf memory".to_string(),
7072 output_preview: "resident completion output".to_string(),
7073 bash_output_list_envelope: None,
7074 output_truncated: false,
7075 original_tokens: None,
7076 compressed_tokens: None,
7077 tokens_skipped: false,
7078 status_reason: None,
7079 live_descendants: Some(Vec::new()),
7080 live_descendants_omitted: 0,
7081 live_descendants_summary: None,
7082 });
7083 let estimate = registry.estimated_memory();
7084 assert!(estimate.estimated_bytes.unwrap() > 0);
7085 assert_eq!(estimate.counts["completion_caches"], 1);
7086 assert_eq!(estimate.counts["sessions"], 1);
7087 }
7088
7089 #[test]
7090 fn gh_structured_detection_rejects_piped_commands() {
7091 assert!(is_gh_structured_command(
7092 "gh issue list --json number,title"
7093 ));
7094 assert!(is_gh_structured_command(
7095 "cd repo && gh issue list --json number,title"
7096 ));
7097
7098 assert!(!is_gh_structured_command(
7099 "gh issue list --json number,title | jq '.[]'"
7100 ));
7101 assert!(!is_gh_structured_command(
7102 "gh issue list --json number,title |"
7103 ));
7104 }
7105
7106 fn insert_terminal_piped_task(
7107 registry: &BgTaskRegistry,
7108 dir: &tempfile::TempDir,
7109 command: &str,
7110 stdout: &str,
7111 stderr: &str,
7112 compressed: bool,
7113 ) -> (String, Arc<BgTask>) {
7114 let task_id = random_slug();
7115 let paths = task_paths(dir.path(), "session", &task_id).unwrap();
7116 fs::create_dir_all(&paths.dir).unwrap();
7117 fs::write(&paths.stdout, stdout).unwrap();
7118 fs::write(&paths.stderr, stderr).unwrap();
7119 let mut metadata = PersistedTask::starting(
7120 task_id.clone(),
7121 "session".to_string(),
7122 command.to_string(),
7123 dir.path().to_path_buf(),
7124 Some(dir.path().to_path_buf()),
7125 Some(30_000),
7126 true,
7127 compressed,
7128 );
7129 metadata.mark_terminal(BgTaskStatus::Completed, Some(0), None);
7130 write_task(&paths.json, &metadata).unwrap();
7131 registry
7132 .insert_rehydrated_task(metadata, paths, true)
7133 .expect("insert terminal task");
7134 let task = registry.task_for_session(&task_id, "session").unwrap();
7135 (task_id, task)
7136 }
7137
7138 #[test]
7139 fn bash_zero_preview_running_status_skips_output_read_while_explicit_preview_reads() {
7140 let registry = BgTaskRegistry::default();
7141 let dir = tempfile::tempdir().unwrap();
7142 let task_id = random_slug();
7143 let paths = task_paths(dir.path(), "session", &task_id).unwrap();
7144 fs::create_dir_all(&paths.dir).unwrap();
7145 fs::write(&paths.stdout, "live output\n").unwrap();
7146 fs::write(&paths.stderr, "").unwrap();
7147 let stdout_path = paths.stdout.clone();
7148 let mut metadata = PersistedTask::starting(
7149 task_id.clone(),
7150 "session".to_string(),
7151 "sleep 60".to_string(),
7152 dir.path().to_path_buf(),
7153 Some(dir.path().to_path_buf()),
7154 Some(30_000),
7155 true,
7156 false,
7157 );
7158 metadata.status = BgTaskStatus::Running;
7159 write_task(&paths.json, &metadata).unwrap();
7160 registry
7161 .insert_rehydrated_task(metadata, paths, false)
7162 .expect("insert running task");
7163
7164 crate::bash_background::buffer::reset_tail_read_count(&stdout_path);
7165 for _ in 0..5 {
7166 let snapshot = registry
7167 .status(&task_id, "session", Some(dir.path()), Some(dir.path()), 0)
7168 .expect("running snapshot");
7169 assert_eq!(snapshot.info.status, BgTaskStatus::Running);
7170 assert!(snapshot.output_preview.is_empty());
7171 }
7172 assert_eq!(
7173 crate::bash_background::buffer::tail_read_count(&stdout_path),
7174 0
7175 );
7176
7177 let snapshot = registry
7178 .status(
7179 &task_id,
7180 "session",
7181 Some(dir.path()),
7182 Some(dir.path()),
7183 RUNNING_OUTPUT_PREVIEW_BYTES,
7184 )
7185 .expect("explicit running snapshot");
7186 assert_eq!(snapshot.output_preview, "live output\n");
7187 assert_eq!(
7188 crate::bash_background::buffer::tail_read_count(&stdout_path),
7189 1
7190 );
7191 }
7192
7193 #[test]
7194 fn artifact_read_capability_requires_exact_canonical_path_and_session() {
7195 let registry = BgTaskRegistry::default();
7196 let dir = tempfile::tempdir().unwrap();
7197 let (_task_id, task) = insert_terminal_piped_task(
7198 ®istry,
7199 &dir,
7200 "printf output",
7201 "stdout\n",
7202 "stderr\n",
7203 true,
7204 );
7205 fs::write(&task.paths.exit, "0\n").unwrap();
7206
7207 assert!(registry.is_session_owned_artifact_path("session", &task.paths.stdout));
7208 assert!(registry.is_session_owned_artifact_path("session", &task.paths.stderr));
7209 assert!(registry.is_session_owned_artifact_path("session", &task.paths.exit));
7210 assert!(!registry.is_session_owned_artifact_path("different-session", &task.paths.stdout));
7211 assert!(!registry.is_session_owned_artifact_path("session", &task.paths.json));
7212
7213 let unregistered = task.paths.dir.join("unregistered-output");
7214 fs::write(&unregistered, "not a task artifact\n").unwrap();
7215 assert!(!registry.is_session_owned_artifact_path("session", &unregistered));
7216 }
7217
7218 #[cfg(unix)]
7219 #[test]
7220 fn artifact_directory_symlink_does_not_create_a_prefix_exception() {
7221 let registry = BgTaskRegistry::default();
7222 let dir = tempfile::tempdir().unwrap();
7223 let project = dir.path().join("project");
7224 fs::create_dir_all(&project).unwrap();
7225 let (_task_id, task) =
7226 insert_terminal_piped_task(®istry, &dir, "printf output", "stdout\n", "", true);
7227 let link = project.join("task-artifacts");
7228 std::os::unix::fs::symlink(&task.paths.dir, &link).unwrap();
7229 let unregistered = task.paths.dir.join("unregistered-output");
7230 fs::write(&unregistered, "not registered\n").unwrap();
7231
7232 assert!(!registry.is_session_owned_artifact_path("session", &link));
7233 assert!(
7234 !registry.is_session_owned_artifact_path("session", &link.join("unregistered-output"))
7235 );
7236 assert!(registry.is_session_owned_artifact_path(
7237 "session",
7238 &link.join(task.paths.stdout.file_name().unwrap())
7239 ));
7240
7241 let outside = dir.path().join("outside-secret");
7242 fs::write(&outside, "must stay private\n").unwrap();
7243 fs::remove_file(&task.paths.stdout).unwrap();
7244 std::os::unix::fs::symlink(&outside, &task.paths.stdout).unwrap();
7245 assert!(!registry.is_session_owned_artifact_path("session", &task.paths.stdout));
7246 }
7247
7248 #[test]
7249 fn recovery_footer_uses_bash_status_when_artifact_is_not_registered() {
7250 let registry = BgTaskRegistry::default();
7251 let dir = tempfile::tempdir().unwrap();
7252 let task_id = "bash-1111111111111111";
7253 let paths = task_paths(dir.path(), "session", task_id).unwrap();
7254 fs::create_dir_all(&paths.dir).unwrap();
7255 fs::write(
7256 &paths.stdout,
7257 format!("{}tail\n", "output-line\n".repeat(2_000)),
7258 )
7259 .unwrap();
7260 fs::write(&paths.stderr, "").unwrap();
7261 let mut metadata = PersistedTask::starting(
7262 task_id.to_string(),
7263 "session".to_string(),
7264 "printf output".to_string(),
7265 dir.path().to_path_buf(),
7266 Some(dir.path().to_path_buf()),
7267 Some(30_000),
7268 true,
7269 true,
7270 );
7271 metadata.mark_terminal(BgTaskStatus::Completed, Some(0), None);
7272 write_task(&paths.json, &metadata).unwrap();
7273
7274 let cache = registry
7275 .render_terminal_output_from_paths(&metadata, &paths)
7276 .expect("terminal render");
7277
7278 assert!(cache
7279 .output_preview
7280 .contains("use bash_status({taskId: \"bash-1111111111111111\"})"));
7281 assert!(!cache.output_preview.contains("full output: read "));
7282 }
7283
7284 fn insert_terminal_pty_task(
7285 registry: &BgTaskRegistry,
7286 dir: &tempfile::TempDir,
7287 pty_output: &str,
7288 ) -> (String, Arc<BgTask>) {
7289 let task_id = random_slug();
7290 let paths = task_paths(dir.path(), "session", &task_id).unwrap();
7291 fs::create_dir_all(&paths.dir).unwrap();
7292 fs::write(&paths.pty, pty_output).unwrap();
7293 let mut metadata = PersistedTask::starting(
7294 task_id.clone(),
7295 "session".to_string(),
7296 "python".to_string(),
7297 dir.path().to_path_buf(),
7298 Some(dir.path().to_path_buf()),
7299 Some(30_000),
7300 true,
7301 true,
7302 );
7303 metadata.mode = BgMode::Pty;
7304 metadata.mark_terminal(BgTaskStatus::Completed, Some(0), None);
7305 write_task(&paths.json, &metadata).unwrap();
7306 registry
7307 .insert_rehydrated_task(metadata, paths, true)
7308 .expect("insert terminal pty task");
7309 let task = registry.task_for_session(&task_id, "session").unwrap();
7310 (task_id, task)
7311 }
7312
7313 #[cfg(unix)]
7314 fn wait_for_terminal_snapshot(
7315 registry: &BgTaskRegistry,
7316 task_id: &str,
7317 session_id: &str,
7318 project: &Path,
7319 storage: &Path,
7320 ) -> BgTaskSnapshot {
7321 let started = Instant::now();
7322 loop {
7323 let snapshot = registry
7324 .status(task_id, session_id, Some(project), Some(storage), 4096)
7325 .expect("spawned task should be visible to status");
7326 if snapshot.info.status.is_terminal() {
7327 return snapshot;
7328 }
7329 assert!(
7330 started.elapsed() < Duration::from_secs(10),
7331 "timed out waiting for task {task_id} to finish; last status={:?}",
7332 snapshot.info.status
7333 );
7334 std::thread::sleep(Duration::from_millis(50));
7335 }
7336 }
7337
7338 fn write_running_project_task(storage: &Path, project: &Path, session: &str, task_id: &str) {
7339 let paths = task_paths(storage, session, task_id).unwrap();
7340 let mut metadata = PersistedTask::starting(
7341 task_id.to_string(),
7342 session.to_string(),
7343 "sleep 60".to_string(),
7344 project.to_path_buf(),
7345 Some(project.to_path_buf()),
7346 Some(30_000),
7347 true,
7348 true,
7349 );
7350 metadata.status = BgTaskStatus::Running;
7351 metadata.child_pid = Some(std::process::id());
7357 write_task(&paths.json, &metadata).unwrap();
7358 fs::write(&paths.stdout, "still running\n").unwrap();
7359 fs::write(&paths.stderr, "").unwrap();
7360 }
7361
7362 #[test]
7363 fn status_replay_filters_same_session_by_project_root() {
7364 let project_a = tempfile::tempdir().unwrap();
7365 let project_b = tempfile::tempdir().unwrap();
7366 let storage = tempfile::tempdir().unwrap();
7367 let session = "shared-session";
7368 let task_id = "bash-2222222222222222";
7369 write_running_project_task(storage.path(), project_a.path(), session, task_id);
7370
7371 let actor_b = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
7372 assert!(actor_b
7373 .status(
7374 task_id,
7375 session,
7376 Some(project_b.path()),
7377 Some(storage.path()),
7378 1024,
7379 )
7380 .is_none());
7381 assert!(actor_b.task_for_session(task_id, session).is_none());
7382
7383 let actor_a = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
7384 let snapshot = actor_a
7385 .status(
7386 task_id,
7387 session,
7388 Some(project_a.path()),
7389 Some(storage.path()),
7390 1024,
7391 )
7392 .expect("owning project should replay its task");
7393 assert_eq!(snapshot.info.status, BgTaskStatus::Running);
7394 }
7395
7396 #[cfg(unix)]
7397 #[test]
7398 fn multiline_pipeline_stdout_persists_all_lines_after_terminal_status() {
7399 let cases = [
7400 (
7401 "long-first",
7402 "sleep 0.5; printf 'one\\n' | cat\nprintf 'two\\n' | grep -c two\nprintf 'three\\n' | cat",
7403 vec!["one", "1", "three"],
7404 ),
7405 (
7406 "short-first",
7407 "printf 'one\\n' | cat\nsleep 0.2; printf 'two\\n' | grep -c two\nprintf 'three\\n' | cat",
7408 vec!["one", "1", "three"],
7409 ),
7410 (
7411 "failing-middle",
7412 "sleep 0.2; printf 'one\\n' | cat\nfalse; printf 'after-false\\n' | cat\nprintf 'three\\n' | cat",
7413 vec!["one", "after-false", "three"],
7414 ),
7415 ];
7416
7417 for (name, command, expected_lines) in cases {
7418 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
7419 let dir = tempfile::tempdir().unwrap();
7420 let session_id = format!("session-{name}");
7421 let task_id = registry
7422 .spawn(
7423 SpawnPlan::Unsandboxed,
7424 command,
7425 session_id.clone(),
7426 dir.path().to_path_buf(),
7427 HashMap::new(),
7428 Some(Duration::from_secs(30)),
7429 dir.path().to_path_buf(),
7430 10,
7431 true,
7432 true,
7433 Some(dir.path().to_path_buf()),
7434 )
7435 .unwrap();
7436
7437 let snapshot = wait_for_terminal_snapshot(
7438 ®istry,
7439 &task_id,
7440 &session_id,
7441 dir.path(),
7442 dir.path(),
7443 );
7444 assert_eq!(
7445 snapshot.info.status,
7446 BgTaskStatus::Completed,
7447 "{name}: task should complete; snapshot={snapshot:?}"
7448 );
7449 assert_eq!(
7450 snapshot.exit_code,
7451 Some(0),
7452 "{name}: script should use the final command's exit code"
7453 );
7454
7455 let stdout = String::from_utf8(
7456 registry
7457 .read_artifact(&task_id, &session_id, TaskArtifact::Stdout)
7458 .expect("read validated stdout artifact"),
7459 )
7460 .expect("stdout is UTF-8");
7461 let lines: Vec<&str> = stdout.lines().collect();
7462 assert_eq!(
7463 lines, expected_lines,
7464 "{name}: raw stdout artifact must include every newline-separated command's output"
7465 );
7466 }
7467 }
7468
7469 #[test]
7470 fn recognizes_all_recovery_marker_forms() {
7471 assert!(is_recovery_marker(
7472 "[truncated output; full output: read \"/tmp/out\"]"
7473 ));
7474 assert!(is_recovery_marker(
7475 "[omitted output; see remaining: tail -n +42 \"/tmp/out\"]"
7476 ));
7477 assert!(is_recovery_marker(
7478 "[truncated output; full output unavailable]"
7479 ));
7480 assert!(is_recovery_marker(
7481 r#"[truncated 123 bytes from saved output prefix; retained output: read "/tmp/out"]"#
7482 ));
7483 }
7484
7485 #[test]
7486 fn recovery_marker_reports_disk_prefix_truncation_as_retained_output() {
7487 let recovery = RecoveryContext {
7488 dropped_by_class: BTreeMap::new(),
7489 had_inner_drop: false,
7490 offset_hint_eligible: false,
7491 offset_start_line: None,
7492 byte_truncated: false,
7493 disk_truncated_prefix_bytes: 4096,
7494 output_path: Some("/tmp/stdout".to_string()),
7495 stderr_path: None,
7496 include_stderr_path: false,
7497 artifact_access: ArtifactRecoveryAccess {
7498 task_id: "bash-test".to_string(),
7499 readable: true,
7500 },
7501 };
7502
7503 let marker = recovery_marker(&recovery).expect("disk truncation must emit marker");
7504
7505 assert!(marker.contains("truncated 4096 bytes from saved output prefix"));
7506 assert!(marker.contains(r#"retained output: read "/tmp/stdout""#));
7507 assert!(!marker.contains("full output: read"));
7508 }
7509
7510 #[test]
7511 fn killed_exit_marker_sets_nonzero_sentinel_exit_code() {
7512 let metadata = PersistedTask::starting(
7513 "task".to_string(),
7514 "session".to_string(),
7515 "cargo test".to_string(),
7516 PathBuf::from("/tmp"),
7517 None,
7518 None,
7519 true,
7520 true,
7521 );
7522
7523 let terminal = terminal_metadata_from_marker(metadata, ExitMarker::Killed, None);
7524
7525 assert_eq!(terminal.status, BgTaskStatus::Killed);
7526 assert_eq!(terminal.exit_code, Some(137));
7527 }
7528
7529 #[test]
7530 fn terminal_status_polls_use_cached_render_once_and_off_lock() {
7531 let registry = BgTaskRegistry::default();
7532 let dir = tempfile::tempdir().unwrap();
7533 let (_task_id, task) = insert_terminal_piped_task(
7534 ®istry,
7535 &dir,
7536 "custom-tool --verbose",
7537 &"stdout line\n".repeat(200_000),
7538 "",
7539 true,
7540 );
7541 let calls = Arc::new(AtomicUsize::new(0));
7542 let saw_unlocked_state = Arc::new(AtomicBool::new(false));
7543 let task_holder = Arc::new(Mutex::new(Some(Arc::clone(&task))));
7544 let calls_for_closure = Arc::clone(&calls);
7545 let unlocked_for_closure = Arc::clone(&saw_unlocked_state);
7546 let task_for_closure = Arc::clone(&task_holder);
7547 registry.set_compressor_with_exit_code(move |_command, output, _exit_code| {
7548 calls_for_closure.fetch_add(1, Ordering::SeqCst);
7549 if let Some(task) = task_for_closure.lock().unwrap().as_ref() {
7550 if task.state.try_lock().is_ok() {
7551 unlocked_for_closure.store(true, Ordering::SeqCst);
7552 }
7553 }
7554 CompressionResult::new(format!("compressed {} bytes", output.len()))
7555 });
7556
7557 let first = registry
7558 .status(
7559 &task.task_id,
7560 "session",
7561 None,
7562 Some(dir.path()),
7563 RUNNING_OUTPUT_PREVIEW_BYTES,
7564 )
7565 .unwrap();
7566 let second = registry
7567 .status(
7568 &task.task_id,
7569 "session",
7570 None,
7571 Some(dir.path()),
7572 RUNNING_OUTPUT_PREVIEW_BYTES,
7573 )
7574 .unwrap();
7575 let listed = registry.list(RUNNING_OUTPUT_PREVIEW_BYTES);
7576
7577 assert_eq!(
7578 calls.load(Ordering::SeqCst),
7579 1,
7580 "terminal render must be cached"
7581 );
7582 assert!(
7583 saw_unlocked_state.load(Ordering::SeqCst),
7584 "compressor must run after releasing the task state lock"
7585 );
7586 assert!(first.output_preview.starts_with("compressed "));
7587 assert_eq!(second.output_preview, first.output_preview);
7588 assert_eq!(listed[0].output_preview, first.output_preview);
7589 }
7590
7591 #[test]
7592 fn completion_preview_success_keeps_tail_only() {
7593 let registry = BgTaskRegistry::default();
7598 let dir = tempfile::tempdir().unwrap();
7599 let output = format!("HEAD-SIGNAL\n{}TAIL-SIGNAL\n", "middle\n".repeat(2_000));
7600 let (_task_id, task) =
7601 insert_terminal_piped_task(®istry, &dir, "cat big.log", &output, "", false);
7602
7603 registry.post_terminal_transition(&task, true).unwrap();
7604 let completions = registry.drain_completions_for_session(Some("session"));
7605 assert_eq!(completions.len(), 1);
7606 let preview = &completions[0].output_preview;
7607 assert!(preview.contains("TAIL-SIGNAL"), "preview was {preview:?}");
7608 assert!(!preview.contains("HEAD-SIGNAL"), "preview was {preview:?}");
7609 assert!(completions[0].output_truncated);
7610 }
7611
7612 #[test]
7613 fn completion_preview_failure_keeps_head_and_tail() {
7614 let registry = BgTaskRegistry::default();
7617 let dir = tempfile::tempdir().unwrap();
7618 let output = format!("HEAD-SIGNAL\n{}TAIL-SIGNAL\n", "middle\n".repeat(2_000));
7619 let task_id = random_slug();
7620 let paths = task_paths(dir.path(), "session", &task_id).unwrap();
7621 fs::create_dir_all(&paths.dir).unwrap();
7622 fs::write(&paths.stdout, &output).unwrap();
7623 fs::write(&paths.stderr, "").unwrap();
7624 let mut metadata = PersistedTask::starting(
7625 task_id.clone(),
7626 "session".to_string(),
7627 "cat big.log".to_string(),
7628 dir.path().to_path_buf(),
7629 Some(dir.path().to_path_buf()),
7630 Some(30_000),
7631 true,
7632 false,
7633 );
7634 metadata.mark_terminal(BgTaskStatus::Failed, Some(1), None);
7635 write_task(&paths.json, &metadata).unwrap();
7636 registry
7637 .insert_rehydrated_task(metadata, paths, true)
7638 .expect("insert terminal task");
7639 let task = registry.task_for_session(&task_id, "session").unwrap();
7640
7641 registry.post_terminal_transition(&task, true).unwrap();
7642 let completions = registry.drain_completions_for_session(Some("session"));
7643 assert_eq!(completions.len(), 1);
7644 let preview = &completions[0].output_preview;
7645 assert!(preview.contains("HEAD-SIGNAL"), "preview was {preview:?}");
7646 assert!(preview.contains("TAIL-SIGNAL"), "preview was {preview:?}");
7647 }
7648
7649 #[test]
7650 fn has_completions_for_session_matches_pending_delivery() {
7651 let registry = BgTaskRegistry::default();
7652 assert!(!registry.has_completions_for_session(Some("session")));
7653 assert!(!registry.has_completions_for_session(None));
7654
7655 let dir = tempfile::tempdir().unwrap();
7656 let (_task_id, task) =
7657 insert_terminal_piped_task(®istry, &dir, QUICK_SUCCESS_COMMAND, "done\n", "", false);
7658 registry.post_terminal_transition(&task, true).unwrap();
7659
7660 assert!(registry.has_completions_for_session(Some("session")));
7661 assert!(registry.has_completions_for_session(None));
7662 assert!(!registry.has_completions_for_session(Some("other-session")));
7663
7664 let completions = registry.drain_completions_for_session(Some("session"));
7665 assert_eq!(completions.len(), 1);
7666 assert_eq!(completions[0].task_id, task.task_id);
7667 }
7668
7669 #[test]
7670 fn completion_drain_redelivers_until_ack_marks_it_delivered() {
7671 let registry = BgTaskRegistry::default();
7672 let dir = tempfile::tempdir().unwrap();
7673 let (task_id, task) =
7674 insert_terminal_piped_task(®istry, &dir, QUICK_SUCCESS_COMMAND, "done\n", "", false);
7675 registry.post_terminal_transition(&task, true).unwrap();
7676
7677 let first = registry.drain_completions_for_session(Some("session"));
7678 let second = registry.drain_completions_for_session(Some("session"));
7679 assert_eq!(first.len(), 1);
7680 assert_eq!(second.len(), 1);
7681 assert_eq!(first[0].task_id, task_id);
7682 assert_eq!(second[0].task_id, task_id);
7683
7684 let resolved =
7685 resolve_task_layout(&session_tasks_dir(dir.path(), "session"), &task_id).unwrap();
7686 assert!(!read_task_at(&resolved).unwrap().completion_delivered);
7687
7688 assert_eq!(
7689 registry.ack_completions_for_session(Some("session"), std::slice::from_ref(&task_id)),
7690 vec![task_id.clone()]
7691 );
7692 assert!(registry
7693 .drain_completions_for_session(Some("session"))
7694 .is_empty());
7695 assert!(read_task_at(&resolved).unwrap().completion_delivered);
7696 }
7697
7698 #[test]
7699 fn structured_gh_json_survives_intact_and_ignores_stderr() {
7700 let registry = BgTaskRegistry::default();
7701 let dir = tempfile::tempdir().unwrap();
7702 let calls = Arc::new(AtomicUsize::new(0));
7703 let calls_for_closure = Arc::clone(&calls);
7704 registry.set_compressor_with_exit_code(move |_command, output, _exit_code| {
7705 calls_for_closure.fetch_add(1, Ordering::SeqCst);
7706 CompressionResult::new(output)
7707 });
7708 let (task_id, _task) = insert_terminal_piped_task(
7709 ®istry,
7710 &dir,
7711 "gh pr view 123 --json body",
7712 "{\"body\":\"hello\"}",
7713 "warning: stderr must not join json",
7714 true,
7715 );
7716
7717 let snapshot = registry
7718 .status(
7719 &task_id,
7720 "session",
7721 None,
7722 Some(dir.path()),
7723 RUNNING_OUTPUT_PREVIEW_BYTES,
7724 )
7725 .unwrap();
7726
7727 assert_eq!(snapshot.output_preview, "{\"body\":\"hello\"}");
7728 assert!(!snapshot.output_preview.contains("warning"));
7729 assert!(!snapshot.output_truncated);
7730 assert_eq!(
7731 calls.load(Ordering::SeqCst),
7732 0,
7733 "structured JSON bypasses compression"
7734 );
7735 }
7736
7737 #[test]
7738 fn registry_emits_single_recovery_marker_for_class_drops() {
7739 let registry = BgTaskRegistry::default();
7740 let dir = tempfile::tempdir().unwrap();
7741 registry.set_compressor_with_exit_code(move |_command, _output, _exit_code| {
7742 let mut dropped = BTreeMap::new();
7743 dropped.insert(DropClass::Error, 18);
7744 dropped.insert(DropClass::Warning, 6);
7745 CompressionResult::with_class_drops("kept diagnostic", dropped)
7746 });
7747 let (task_id, task) =
7748 insert_terminal_piped_task(®istry, &dir, "custom-tool", "raw", "", true);
7749
7750 let snapshot = registry
7751 .status(
7752 &task_id,
7753 "session",
7754 None,
7755 Some(dir.path()),
7756 RUNNING_OUTPUT_PREVIEW_BYTES,
7757 )
7758 .unwrap();
7759
7760 assert_eq!(snapshot.output_preview.matches("full output:").count(), 1);
7761 assert!(snapshot.output_preview.contains("+18 more errors"));
7762 assert!(snapshot.output_preview.contains("+6 more warnings"));
7763 assert!(snapshot
7764 .output_preview
7765 .contains(&format!("read \"{}\"", task.paths.stdout.display())));
7766 assert!(!snapshot.output_preview.contains("tail -n +"));
7767 assert!(snapshot.output_truncated);
7768 }
7769
7770 #[test]
7771 fn registry_marker_reports_semantic_and_byte_drops_once() {
7772 let registry = BgTaskRegistry::default();
7773 let dir = tempfile::tempdir().unwrap();
7774 registry.set_compressor_with_exit_code(move |_command, _output, _exit_code| {
7775 let mut dropped = BTreeMap::new();
7776 dropped.insert(DropClass::Error, 1);
7777 CompressionResult::with_class_drops(
7778 format!("HEAD-SIGNAL\n{}TAIL-SIGNAL", "middle\n".repeat(8_000)),
7779 dropped,
7780 )
7781 });
7782 let (task_id, _task) =
7783 insert_terminal_piped_task(®istry, &dir, "custom-tool", "raw", "", true);
7784
7785 let snapshot = registry
7786 .status(
7787 &task_id,
7788 "session",
7789 None,
7790 Some(dir.path()),
7791 RUNNING_OUTPUT_PREVIEW_BYTES,
7792 )
7793 .unwrap();
7794
7795 assert_eq!(snapshot.output_preview.matches("full output:").count(), 1);
7796 assert!(snapshot.output_preview.contains("+1 more error"));
7797 assert!(snapshot.output_preview.contains("truncated output"));
7798 assert!(snapshot.output_preview.contains("HEAD-SIGNAL"));
7799 assert!(snapshot.output_preview.contains("TAIL-SIGNAL"));
7800 assert!(!snapshot.output_preview.contains("...<truncated"));
7801 assert!(snapshot.output_truncated);
7802 }
7803
7804 #[test]
7805 fn cargo_stderr_class_drops_name_both_capture_paths() {
7806 let registry = BgTaskRegistry::default();
7807 let dir = tempfile::tempdir().unwrap();
7808 let filter_registry = crate::compress::toml_filter::FilterRegistry::default();
7809 registry.set_compressor_with_exit_code(move |command, output, exit_code| {
7810 crate::compress::compress_with_registry_exit_code(
7811 command,
7812 &output,
7813 exit_code,
7814 &filter_registry,
7815 )
7816 });
7817 let stderr = (0..22)
7818 .map(|index| {
7819 format!(
7820 "error: cargo failure {index}\n --> src/lib.rs:{}:1\n |\n{} | boom\n",
7821 index + 1,
7822 index + 1
7823 )
7824 })
7825 .collect::<Vec<_>>()
7826 .join("\n");
7827 let (task_id, task) = insert_terminal_piped_task(
7828 ®istry,
7829 &dir,
7830 "cargo check",
7831 "Finished dev [unoptimized] target(s) in 0.01s\n",
7832 &stderr,
7833 true,
7834 );
7835
7836 let snapshot = registry
7837 .status(
7838 &task_id,
7839 "session",
7840 None,
7841 Some(dir.path()),
7842 RUNNING_OUTPUT_PREVIEW_BYTES,
7843 )
7844 .unwrap();
7845
7846 assert!(snapshot.output_preview.contains("+2 more errors"));
7847 assert!(snapshot
7848 .output_preview
7849 .contains(&format!("read \"{}\"", task.paths.stdout.display())));
7850 assert!(snapshot
7851 .output_preview
7852 .contains(&format!("read \"{}\"", task.paths.stderr.display())));
7853 assert!(!snapshot.output_preview.contains("tail -n +"));
7854 }
7855
7856 #[test]
7857 fn over_ceiling_structured_json_uses_pointer_not_partial_json() {
7858 let registry = BgTaskRegistry::default();
7859 let dir = tempfile::tempdir().unwrap();
7860 let body = format!("{{\"body\":\"{}\"}}", "x".repeat(60 * 1024));
7861 let (task_id, task) = insert_terminal_piped_task(
7862 ®istry,
7863 &dir,
7864 "cd /repo && gh pr view 123 --json body",
7865 &body,
7866 "",
7867 true,
7868 );
7869
7870 let snapshot = registry
7871 .status(
7872 &task_id,
7873 "session",
7874 None,
7875 Some(dir.path()),
7876 RUNNING_OUTPUT_PREVIEW_BYTES,
7877 )
7878 .unwrap();
7879
7880 assert!(snapshot.output_preview.starts_with("[JSON output "));
7881 assert!(snapshot
7882 .output_preview
7883 .contains(&task.paths.stdout.display().to_string()));
7884 assert!(!snapshot.output_preview.contains(&"x".repeat(1024)));
7885 assert!(snapshot.output_truncated);
7886 }
7887
7888 #[test]
7889 fn toml_strip_tail_cap_uses_full_output_hint_not_offset_hint() {
7890 let registry = BgTaskRegistry::default();
7891 let dir = tempfile::tempdir().unwrap();
7892 let filter_registry = crate::compress::toml_filter::build_registry(
7893 crate::compress::builtin_filters::ALL,
7894 None,
7895 None,
7896 );
7897 registry.set_compressor_with_exit_code(move |command, output, exit_code| {
7898 crate::compress::compress_with_registry_exit_code(
7899 command,
7900 &output,
7901 exit_code,
7902 &filter_registry,
7903 )
7904 });
7905 let stdout = format!(
7906 "make[1]: Entering directory `/tmp`\n{}",
7907 (0..100)
7908 .map(|index| format!("compile line {index}"))
7909 .collect::<Vec<_>>()
7910 .join("\n")
7911 );
7912 let (task_id, task) =
7913 insert_terminal_piped_task(®istry, &dir, "make all", &stdout, "", true);
7914
7915 let snapshot = registry
7916 .status(
7917 &task_id,
7918 "session",
7919 None,
7920 Some(dir.path()),
7921 RUNNING_OUTPUT_PREVIEW_BYTES,
7922 )
7923 .unwrap();
7924
7925 assert!(snapshot.output_preview.contains("compile line 99"));
7926 assert!(snapshot.output_preview.contains(&format!(
7927 "full output: read \"{}\"",
7928 task.paths.stdout.display()
7929 )));
7930 assert!(!snapshot
7931 .output_preview
7932 .contains(&format!("read \"{}\"", task.paths.stderr.display())));
7933 assert!(!snapshot.output_preview.contains("tail -n +"));
7934 }
7935
7936 #[test]
7937 fn compressed_false_raw_passthrough_uses_wider_head_tail_cap() {
7938 let registry = BgTaskRegistry::default();
7939 let dir = tempfile::tempdir().unwrap();
7940 let output = format!("RAW-HEAD\n{}RAW-TAIL\n", "raw-middle\n".repeat(8_000));
7941 let (task_id, task) =
7942 insert_terminal_piped_task(®istry, &dir, "cat raw.log", &output, "RAW-ERR\n", false);
7943
7944 let snapshot = registry
7945 .status(
7946 &task_id,
7947 "session",
7948 None,
7949 Some(dir.path()),
7950 RUNNING_OUTPUT_PREVIEW_BYTES,
7951 )
7952 .unwrap();
7953
7954 assert!(snapshot.output_preview.contains("RAW-HEAD"));
7955 assert!(snapshot.output_preview.contains("RAW-TAIL"));
7956 assert!(snapshot.output_preview.contains("truncated output"));
7957 assert!(snapshot
7958 .output_preview
7959 .contains(&format!("read \"{}\"", task.paths.stdout.display())));
7960 assert!(snapshot
7961 .output_preview
7962 .contains(&format!("read \"{}\"", task.paths.stderr.display())));
7963 assert!(!snapshot.output_preview.contains("tail -n +"));
7964 assert!(snapshot.output_preview.len() > 16 * 1024);
7965 assert!(snapshot.output_truncated);
7966 }
7967
7968 #[test]
7969 fn pty_terminal_snapshot_bypasses_line_compression() {
7970 let registry = BgTaskRegistry::default();
7971 let dir = tempfile::tempdir().unwrap();
7972 let calls = Arc::new(AtomicUsize::new(0));
7973 let calls_for_closure = Arc::clone(&calls);
7974 registry.set_compressor_with_exit_code(move |_command, output, _exit_code| {
7975 calls_for_closure.fetch_add(1, Ordering::SeqCst);
7976 CompressionResult::new(output)
7977 });
7978 let (task_id, _task) = insert_terminal_pty_task(®istry, &dir, "raw\u{1b}[31m pty bytes");
7979
7980 let snapshot = registry
7981 .status(
7982 &task_id,
7983 "session",
7984 None,
7985 Some(dir.path()),
7986 RUNNING_OUTPUT_PREVIEW_BYTES,
7987 )
7988 .unwrap();
7989
7990 assert_eq!(snapshot.info.mode, BgMode::Pty);
7991 assert_eq!(snapshot.output_preview, "");
7992 assert_eq!(calls.load(Ordering::SeqCst), 0);
7993 }
7994
7995 #[test]
7996 fn pty_dimensions_are_persisted_and_returned_in_snapshot() {
7997 let registry = BgTaskRegistry::default();
7998 let dir = tempfile::tempdir().unwrap();
7999 let task_id = registry
8000 .spawn_pty(
8001 SpawnPlan::Unsandboxed,
8002 QUICK_SUCCESS_COMMAND,
8003 "session".to_string(),
8004 dir.path().to_path_buf(),
8005 HashMap::new(),
8006 Some(Duration::from_secs(30)),
8007 dir.path().to_path_buf(),
8008 10,
8009 true,
8010 false,
8011 Some(dir.path().to_path_buf()),
8012 50,
8013 120,
8014 )
8015 .unwrap();
8016
8017 let resolved =
8018 resolve_task_layout(&session_tasks_dir(dir.path(), "session"), &task_id).unwrap();
8019 let metadata = read_task_at(&resolved).unwrap();
8020 assert_eq!(
8021 metadata.schema_version,
8022 crate::bash_background::persistence::SCHEMA_VERSION
8023 );
8024 assert_eq!(metadata.mode, BgMode::Pty);
8025 assert_eq!(metadata.pty_rows, Some(50));
8026 assert_eq!(metadata.pty_cols, Some(120));
8027
8028 let snapshot = registry
8029 .status(&task_id, "session", None, Some(dir.path()), 1024)
8030 .unwrap();
8031 assert_eq!(snapshot.pty_rows, Some(50));
8032 assert_eq!(snapshot.pty_cols, Some(120));
8033 }
8034
8035 fn spawn_dead_child() -> std::process::Child {
8040 #[cfg(unix)]
8041 let mut cmd = std::process::Command::new("true");
8042 #[cfg(windows)]
8043 let mut cmd = {
8044 let mut c = std::process::Command::new("cmd");
8045 c.args(["/c", "exit", "0"]);
8046 c
8047 };
8048 cmd.stdin(std::process::Stdio::null());
8049 cmd.stdout(std::process::Stdio::null());
8050 cmd.stderr(std::process::Stdio::null());
8051 let mut child = cmd.spawn().expect("spawn replacement child for reap test");
8052 let started = Instant::now();
8061 loop {
8062 match child.try_wait() {
8063 Ok(Some(_)) => break,
8064 Ok(None) => {
8065 if started.elapsed() > CHILD_EXIT_LIVENESS_BOUND {
8066 panic!("dead-child stand-in did not exit within the liveness bound");
8067 }
8068 std::thread::sleep(Duration::from_millis(10));
8069 }
8070 Err(error) => panic!("dead-child try_wait failed: {error}"),
8071 }
8072 }
8073 child
8074 }
8075
8076 #[test]
8077 fn ack_marks_delivered_even_when_completion_was_already_consumed_locally() {
8078 let registry = BgTaskRegistry::default();
8079 let dir = tempfile::tempdir().unwrap();
8080 let task_id = registry
8081 .spawn(
8082 SpawnPlan::Unsandboxed,
8083 LONG_RUNNING_COMMAND,
8084 "session".to_string(),
8085 dir.path().to_path_buf(),
8086 HashMap::new(),
8087 Some(Duration::from_secs(30)),
8088 dir.path().to_path_buf(),
8089 10,
8090 true,
8091 false,
8092 Some(dir.path().to_path_buf()),
8093 )
8094 .unwrap();
8095 registry
8096 .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
8097 .unwrap();
8098 assert_eq!(
8099 registry
8100 .drain_completions_for_session(Some("session"))
8101 .len(),
8102 1
8103 );
8104
8105 registry.inner.completions.lock().unwrap().clear();
8108
8109 assert_eq!(
8110 registry.ack_completions_for_session(Some("session"), std::slice::from_ref(&task_id)),
8111 vec![task_id.clone()]
8112 );
8113 assert!(registry
8114 .drain_completions_for_session(Some("session"))
8115 .is_empty());
8116
8117 let resolved =
8118 resolve_task_layout(&session_tasks_dir(dir.path(), "session"), &task_id).unwrap();
8119 let metadata = read_task_at(&resolved).unwrap();
8120 assert!(metadata.completion_delivered);
8121
8122 let replayed = BgTaskRegistry::default();
8123 replayed
8124 .replay_session_inner(dir.path(), "session", None)
8125 .unwrap();
8126 assert!(replayed
8127 .drain_completions_for_session(Some("session"))
8128 .is_empty());
8129 }
8130
8131 #[test]
8132 fn reclaimed_root_kills_running_task_and_persists_reason() {
8133 let registry = BgTaskRegistry::default();
8134 let root = tempfile::tempdir().unwrap();
8135 let storage = tempfile::tempdir().unwrap();
8136 let task_id = registry
8137 .spawn(
8138 SpawnPlan::Unsandboxed,
8139 LONG_RUNNING_COMMAND,
8140 "session".to_string(),
8141 root.path().to_path_buf(),
8142 HashMap::new(),
8143 Some(Duration::from_secs(30)),
8144 storage.path().to_path_buf(),
8145 10,
8146 true,
8147 false,
8148 Some(root.path().to_path_buf()),
8149 )
8150 .unwrap();
8151 let pid = registry
8152 .status(
8153 &task_id,
8154 "session",
8155 Some(root.path()),
8156 Some(storage.path()),
8157 0,
8158 )
8159 .unwrap()
8160 .child_pid
8161 .unwrap();
8162 assert!(is_process_alive(pid));
8163
8164 assert_eq!(registry.kill_running_tasks_for_root(root.path()), 1);
8165 let deadline = Instant::now() + CHILD_EXIT_LIVENESS_BOUND;
8166 while is_process_alive(pid) {
8167 assert!(
8168 Instant::now() < deadline,
8169 "reclaimed task process survived kill"
8170 );
8171 std::thread::sleep(Duration::from_millis(20));
8172 }
8173
8174 let snapshot = registry
8175 .status(
8176 &task_id,
8177 "session",
8178 Some(root.path()),
8179 Some(storage.path()),
8180 0,
8181 )
8182 .unwrap();
8183 assert_eq!(snapshot.info.status, BgTaskStatus::Killed);
8184 assert_eq!(
8185 snapshot.info.status_reason.as_deref(),
8186 Some(ROOT_RECLAIMED_REASON)
8187 );
8188 let persisted = read_task(
8189 ®istry
8190 .task_json_path(&task_id, "session")
8191 .expect("reclaimed task metadata path"),
8192 )
8193 .expect("persisted reclaimed task");
8194 assert_eq!(
8195 persisted.status_reason.as_deref(),
8196 Some(ROOT_RECLAIMED_REASON)
8197 );
8198 let completion = registry
8199 .drain_completions_for_session(Some("session"))
8200 .pop()
8201 .expect("reclaimed task completion");
8202 assert_eq!(
8203 completion.status_reason.as_deref(),
8204 Some(ROOT_RECLAIMED_REASON)
8205 );
8206 registry.detach();
8207 }
8208
8209 #[test]
8210 fn reclaimed_root_kills_pty_task_and_preserves_reason() {
8211 let registry = BgTaskRegistry::default();
8212 let root = tempfile::tempdir().unwrap();
8213 let storage = tempfile::tempdir().unwrap();
8214 let command = if cfg!(windows) {
8215 "Start-Sleep -Seconds 30"
8216 } else {
8217 "sleep 30"
8218 };
8219 let task_id = registry
8220 .spawn_pty(
8221 SpawnPlan::Unsandboxed,
8222 command,
8223 "session".to_string(),
8224 root.path().to_path_buf(),
8225 HashMap::new(),
8226 Some(Duration::from_secs(60)),
8227 storage.path().to_path_buf(),
8228 10,
8229 true,
8230 false,
8231 Some(root.path().to_path_buf()),
8232 24,
8233 80,
8234 )
8235 .unwrap();
8236 let pid = registry
8237 .status(
8238 &task_id,
8239 "session",
8240 Some(root.path()),
8241 Some(storage.path()),
8242 0,
8243 )
8244 .unwrap()
8245 .child_pid
8246 .unwrap();
8247 assert!(is_process_alive(pid));
8248
8249 assert_eq!(registry.kill_running_tasks_for_root(root.path()), 1);
8250 let deadline = Instant::now() + Duration::from_secs(10);
8251 loop {
8252 let snapshot = registry
8253 .status(
8254 &task_id,
8255 "session",
8256 Some(root.path()),
8257 Some(storage.path()),
8258 0,
8259 )
8260 .unwrap();
8261 if snapshot.info.status.is_terminal() {
8262 assert_eq!(snapshot.info.status, BgTaskStatus::Killed);
8263 assert_eq!(
8264 snapshot.info.status_reason.as_deref(),
8265 Some(ROOT_RECLAIMED_REASON)
8266 );
8267 break;
8268 }
8269 assert!(
8270 Instant::now() < deadline,
8271 "reclaimed PTY task did not terminate"
8272 );
8273 std::thread::sleep(Duration::from_millis(20));
8274 }
8275 assert!(!is_process_alive(pid));
8276 let completion = loop {
8281 if let Some(completion) = registry
8282 .drain_completions_for_session(Some("session"))
8283 .pop()
8284 {
8285 break completion;
8286 }
8287 assert!(
8288 Instant::now() < deadline,
8289 "reclaimed PTY completion never arrived"
8290 );
8291 std::thread::sleep(Duration::from_millis(20));
8292 };
8293 assert_eq!(
8294 completion.status_reason.as_deref(),
8295 Some(ROOT_RECLAIMED_REASON)
8296 );
8297 registry.detach();
8298 }
8299
8300 #[test]
8301 fn register_watch_rejects_unknown_task() {
8302 let registry = BgTaskRegistry::default();
8303
8304 let result = registry.register_watch(
8305 "missing-task".to_string(),
8306 WatchPattern::Substring("READY".into()),
8307 true,
8308 );
8309
8310 assert_eq!(result, Err("task_not_found"));
8311 }
8312
8313 #[test]
8314 fn register_watch_on_terminal_task_scans_existing_output() {
8315 let frames = Arc::new(Mutex::new(Vec::new()));
8316 let captured = Arc::clone(&frames);
8317 let sender: crate::context::ProgressSender = Arc::new(Box::new(move |frame| {
8318 captured.lock().unwrap().push(frame);
8319 })
8320 as Box<dyn Fn(PushFrame) + Send + Sync>);
8321 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(Some(sender))));
8322 let dir = tempfile::tempdir().unwrap();
8323 let task_id = registry
8324 .spawn(
8325 SpawnPlan::Unsandboxed,
8326 LONG_RUNNING_COMMAND,
8327 "session".to_string(),
8328 dir.path().to_path_buf(),
8329 HashMap::new(),
8330 Some(Duration::from_secs(30)),
8331 dir.path().to_path_buf(),
8332 10,
8333 true,
8334 false,
8335 Some(dir.path().to_path_buf()),
8336 )
8337 .unwrap();
8338 registry
8339 .inner
8340 .shutdown
8341 .store(true, std::sync::atomic::Ordering::SeqCst);
8342 let task = registry.task_for_session(&task_id, "session").unwrap();
8343 std::fs::write(&task.paths.stdout, "READY\n").unwrap();
8344 registry
8345 .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
8346 .unwrap();
8347 frames.lock().unwrap().clear();
8348 registry.inner.completions.lock().unwrap().clear();
8349
8350 registry
8351 .register_watch(
8352 task_id.clone(),
8353 WatchPattern::Substring("READY".into()),
8354 true,
8355 )
8356 .unwrap();
8357
8358 let frames = frames.lock().unwrap();
8359 let frame = frames
8360 .iter()
8361 .find_map(|frame| match frame {
8362 PushFrame::BashPatternMatch(frame) => Some(frame),
8363 _ => None,
8364 })
8365 .expect("terminal watch registration should emit pattern frame");
8366 assert_eq!(frame.reason, "pattern_match");
8367 assert_eq!(frame.task_id, task_id);
8368 assert_eq!(frame.session_id, "session");
8369 assert_eq!(frame.match_text, "READY");
8370 assert_eq!(frame.match_offset, 0);
8371 assert_eq!(registry.active_watch_count(&frame.task_id), 0);
8372 let metadata = read_task(&task.paths.json).unwrap();
8373 assert!(
8374 !metadata.completion_delivered,
8375 "terminal pattern notification remains unacked until explicit ack"
8376 );
8377 drop(frames);
8378 assert_eq!(
8379 registry.ack_completions_for_session(Some("session"), std::slice::from_ref(&task_id),),
8380 vec![task_id]
8381 );
8382 assert!(read_task(&task.paths.json).unwrap().completion_delivered);
8383 }
8384
8385 #[test]
8386 fn cleanup_finished_removes_terminal_tasks_older_than_threshold() {
8387 let registry = BgTaskRegistry::default();
8388 let dir = tempfile::tempdir().unwrap();
8389 let task_id = registry
8390 .spawn(
8391 SpawnPlan::Unsandboxed,
8392 QUICK_SUCCESS_COMMAND,
8393 "session".to_string(),
8394 dir.path().to_path_buf(),
8395 HashMap::new(),
8396 Some(Duration::from_secs(30)),
8397 dir.path().to_path_buf(),
8398 10,
8399 true,
8400 false,
8401 Some(dir.path().to_path_buf()),
8402 )
8403 .unwrap();
8404 registry
8405 .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
8406 .unwrap();
8407 let completions = registry.drain_completions_for_session(Some("session"));
8408 assert_eq!(completions.len(), 1);
8409 assert_eq!(
8410 registry.ack_completions_for_session(Some("session"), std::slice::from_ref(&task_id)),
8411 vec![task_id.clone()]
8412 );
8413
8414 registry.cleanup_finished(Duration::ZERO);
8415
8416 assert!(registry.inner.tasks.lock().unwrap().is_empty());
8417 }
8418
8419 #[test]
8420 fn cleanup_finished_retains_undelivered_terminals() {
8421 let registry = BgTaskRegistry::default();
8422 let dir = tempfile::tempdir().unwrap();
8423 let task_id = registry
8424 .spawn(
8425 SpawnPlan::Unsandboxed,
8426 QUICK_SUCCESS_COMMAND,
8427 "session".to_string(),
8428 dir.path().to_path_buf(),
8429 HashMap::new(),
8430 Some(Duration::from_secs(30)),
8431 dir.path().to_path_buf(),
8432 10,
8433 true,
8434 false,
8435 Some(dir.path().to_path_buf()),
8436 )
8437 .unwrap();
8438 registry
8439 .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
8440 .unwrap();
8441
8442 registry.cleanup_finished(Duration::ZERO);
8443
8444 assert!(registry.inner.tasks.lock().unwrap().contains_key(&task_id));
8445 }
8446
8447 #[test]
8455 fn reap_child_marks_failed_when_child_exits_without_exit_marker() {
8456 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
8457 let dir = tempfile::tempdir().unwrap();
8458 let task_id = registry
8459 .spawn(
8460 SpawnPlan::Unsandboxed,
8461 QUICK_SUCCESS_COMMAND,
8462 "session".to_string(),
8463 dir.path().to_path_buf(),
8464 HashMap::new(),
8465 Some(Duration::from_secs(30)),
8466 dir.path().to_path_buf(),
8467 10,
8468 true,
8469 false,
8470 Some(dir.path().to_path_buf()),
8471 )
8472 .unwrap();
8473
8474 let task = registry.task_for_session(&task_id, "session").unwrap();
8475
8476 let started = Instant::now();
8481 loop {
8482 let exited = {
8483 let mut state = task.state.lock().unwrap();
8484 match &mut state.runtime {
8485 TaskRuntime::Piped(Some(child)) => matches!(child.try_wait(), Ok(Some(_))),
8486 _ => true,
8487 }
8488 };
8489 if exited {
8490 break;
8491 }
8492 assert!(
8493 started.elapsed() < CHILD_EXIT_LIVENESS_BOUND,
8494 "child should exit quickly"
8495 );
8496 std::thread::sleep(Duration::from_millis(20));
8497 }
8498
8499 registry
8507 .inner
8508 .shutdown
8509 .store(true, std::sync::atomic::Ordering::SeqCst);
8510 std::thread::sleep(Duration::from_millis(550));
8514
8515 let _ = std::fs::remove_file(&task.paths.exit);
8518
8519 {
8534 let mut state = task.state.lock().unwrap();
8535 state.metadata.status = BgTaskStatus::Running;
8536 state.metadata.status_reason = None;
8537 state.metadata.exit_code = None;
8538 state.metadata.finished_at = None;
8539 state.metadata.duration_ms = None;
8540 crate::bash_background::persistence::write_task(&task.paths.json, &state.metadata)
8543 .expect("persist reset Running metadata for reap_child test");
8544 if matches!(state.runtime, TaskRuntime::Piped(None)) {
8548 state.runtime = TaskRuntime::Piped(Some(spawn_dead_child()));
8549 }
8550 }
8551 *task.terminal_at.lock().unwrap() = None;
8554
8555 assert!(
8558 task.is_running(),
8559 "precondition: metadata.status == Running"
8560 );
8561 assert!(
8562 !task.paths.exit.exists(),
8563 "precondition: exit marker absent"
8564 );
8565
8566 registry.reap_child(&task);
8571
8572 {
8573 let state = task.state.lock().unwrap();
8574 assert_eq!(
8575 state.metadata.status,
8576 BgTaskStatus::Running,
8577 "first reap must leave status Running while waiting one pass for marker"
8578 );
8579 assert_eq!(
8580 state.metadata.status_reason, None,
8581 "first reap must not record a failure reason"
8582 );
8583 assert!(
8584 matches!(state.runtime, TaskRuntime::Piped(None)),
8585 "child handle must be released after first reap"
8586 );
8587 assert!(
8588 state.detached,
8589 "task must be marked detached after first reap"
8590 );
8591 }
8592
8593 registry.reap_child(&task);
8597
8598 let state = task.state.lock().unwrap();
8599 assert!(
8600 state.metadata.status.is_terminal(),
8601 "second reap must transition to terminal when PID dead and no marker. Got status={:?}",
8602 state.metadata.status
8603 );
8604 assert_eq!(
8605 state.metadata.status,
8606 BgTaskStatus::Failed,
8607 "must specifically be Failed (not Killed): status={:?}",
8608 state.metadata.status
8609 );
8610 assert_eq!(
8611 state.metadata.status_reason.as_deref(),
8612 Some("process exited without exit marker"),
8613 "reason must match replay path's wording: {:?}",
8614 state.metadata.status_reason
8615 );
8616 assert!(
8617 matches!(state.runtime, TaskRuntime::Piped(None)),
8618 "child handle must stay released after second reap"
8619 );
8620 assert!(
8621 state.detached,
8622 "task must remain detached after second reap"
8623 );
8624 }
8625
8626 #[test]
8631 fn reap_child_preserves_running_when_exit_marker_exists() {
8632 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
8633 let dir = tempfile::tempdir().unwrap();
8634 let task_id = registry
8635 .spawn(
8636 SpawnPlan::Unsandboxed,
8637 QUICK_SUCCESS_COMMAND,
8638 "session".to_string(),
8639 dir.path().to_path_buf(),
8640 HashMap::new(),
8641 Some(Duration::from_secs(30)),
8642 dir.path().to_path_buf(),
8643 10,
8644 true,
8645 false,
8646 Some(dir.path().to_path_buf()),
8647 )
8648 .unwrap();
8649
8650 let task = registry.task_for_session(&task_id, "session").unwrap();
8651
8652 let started = Instant::now();
8655 loop {
8656 let exited = {
8657 let mut state = task.state.lock().unwrap();
8658 match &mut state.runtime {
8659 TaskRuntime::Piped(Some(child)) => matches!(child.try_wait(), Ok(Some(_))),
8660 _ => true,
8661 }
8662 };
8663 if exited && task.paths.exit.exists() {
8664 break;
8665 }
8666 assert!(
8667 started.elapsed() < CHILD_EXIT_LIVENESS_BOUND,
8668 "child should exit and write marker quickly"
8669 );
8670 std::thread::sleep(Duration::from_millis(20));
8671 }
8672
8673 registry
8679 .inner
8680 .shutdown
8681 .store(true, std::sync::atomic::Ordering::SeqCst);
8682 std::thread::sleep(Duration::from_millis(550));
8683
8684 {
8690 let mut state = task.state.lock().unwrap();
8691 state.metadata.status = BgTaskStatus::Running;
8692 state.metadata.status_reason = None;
8693 if matches!(state.runtime, TaskRuntime::Piped(None)) {
8694 state.runtime = TaskRuntime::Piped(Some(spawn_dead_child()));
8695 }
8696 }
8697 *task.terminal_at.lock().unwrap() = None;
8698 if !task.paths.exit.exists() {
8701 std::fs::write(&task.paths.exit, "0").expect("write replacement exit marker");
8702 }
8703
8704 registry.reap_child(&task);
8708
8709 let state = task.state.lock().unwrap();
8710 assert!(
8711 matches!(state.runtime, TaskRuntime::Piped(None)),
8712 "child handle still released even when marker exists"
8713 );
8714 assert!(
8715 state.detached,
8716 "task still marked detached even when marker exists"
8717 );
8718 assert_eq!(
8723 state.metadata.status,
8724 BgTaskStatus::Running,
8725 "reap_child must defer to poll_task when marker exists"
8726 );
8727 }
8728
8729 #[cfg(unix)]
8733 fn pid_stat(pid: u32) -> Option<String> {
8734 let output = std::process::Command::new("ps")
8735 .args(["-o", "stat=", "-p", &pid.to_string()])
8736 .output()
8737 .ok()?;
8738 if !output.status.success() {
8739 return None;
8740 }
8741 let stat = String::from_utf8_lossy(&output.stdout).trim().to_string();
8742 if stat.is_empty() {
8743 None
8744 } else {
8745 Some(stat)
8746 }
8747 }
8748
8749 #[cfg(unix)]
8751 fn is_zombie(pid: u32) -> bool {
8752 pid_stat(pid).is_some_and(|stat| stat.starts_with('Z'))
8753 }
8754
8755 #[cfg(unix)]
8761 fn spawn_unreaped_zombie() -> std::process::Child {
8762 let child = std::process::Command::new("true")
8763 .stdin(std::process::Stdio::null())
8764 .stdout(std::process::Stdio::null())
8765 .stderr(std::process::Stdio::null())
8766 .spawn()
8767 .expect("spawn zombie stand-in");
8768 let pid = child.id();
8769 let started = Instant::now();
8770 while !is_zombie(pid) {
8771 assert!(
8772 started.elapsed() < CHILD_EXIT_LIVENESS_BOUND,
8773 "stand-in child should become a zombie within the liveness bound"
8774 );
8775 std::thread::sleep(Duration::from_millis(10));
8776 }
8777 child
8779 }
8780
8781 #[cfg(unix)]
8791 #[test]
8792 fn finalize_from_marker_reaps_child_no_zombie() {
8793 use std::sync::atomic::Ordering;
8794
8795 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
8796 let dir = tempfile::tempdir().unwrap();
8797 let task_id = registry
8798 .spawn(
8799 SpawnPlan::Unsandboxed,
8800 QUICK_SUCCESS_COMMAND,
8801 "session".to_string(),
8802 dir.path().to_path_buf(),
8803 HashMap::new(),
8804 Some(Duration::from_secs(30)),
8805 dir.path().to_path_buf(),
8806 10,
8807 true,
8808 false,
8809 Some(dir.path().to_path_buf()),
8810 )
8811 .unwrap();
8812
8813 registry.inner.shutdown.store(true, Ordering::SeqCst);
8817 std::thread::sleep(Duration::from_millis(550));
8818
8819 let task = registry.task_for_session(&task_id, "session").unwrap();
8820
8821 let started = Instant::now();
8825 while !task.paths.exit.exists() {
8826 assert!(
8827 started.elapsed() < CHILD_EXIT_LIVENESS_BOUND,
8828 "exit marker should land quickly for `true`"
8829 );
8830 std::thread::sleep(Duration::from_millis(20));
8831 }
8832
8833 let zombie_pid;
8839 {
8840 let mut state = task.state.lock().unwrap();
8841 state.metadata.status = BgTaskStatus::Running;
8842 state.metadata.status_reason = None;
8843 state.metadata.exit_code = None;
8844 state.metadata.finished_at = None;
8845 state.metadata.duration_ms = None;
8846 crate::bash_background::persistence::write_task(&task.paths.json, &state.metadata)
8847 .expect("persist reset Running metadata");
8848 let zombie = spawn_unreaped_zombie();
8849 zombie_pid = zombie.id();
8850 state.runtime = TaskRuntime::Piped(Some(zombie));
8851 }
8852 *task.terminal_at.lock().unwrap() = None;
8853
8854 assert!(
8856 is_zombie(zombie_pid),
8857 "precondition: stand-in child {zombie_pid} must be a zombie before finalize"
8858 );
8859
8860 registry.poll_task(&task).unwrap();
8863
8864 {
8865 let state = task.state.lock().unwrap();
8866 assert!(
8867 matches!(state.runtime, TaskRuntime::Piped(None)),
8868 "child handle must be released after marker finalize"
8869 );
8870 assert!(
8871 state.metadata.status.is_terminal(),
8872 "task must be terminal after marker finalize: {:?}",
8873 state.metadata.status
8874 );
8875 }
8876
8877 assert!(
8880 !is_zombie(zombie_pid),
8881 "issue #91 regression: child {zombie_pid} left as <defunct> zombie \
8882 after the exit-marker terminal transition"
8883 );
8884 }
8885
8886 #[cfg(unix)]
8890 #[test]
8891 fn kill_with_existing_marker_reaps_child_no_zombie() {
8892 use std::sync::atomic::Ordering;
8893
8894 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
8895 let dir = tempfile::tempdir().unwrap();
8896 let task_id = registry
8897 .spawn(
8898 SpawnPlan::Unsandboxed,
8899 QUICK_SUCCESS_COMMAND,
8900 "session".to_string(),
8901 dir.path().to_path_buf(),
8902 HashMap::new(),
8903 Some(Duration::from_secs(30)),
8904 dir.path().to_path_buf(),
8905 10,
8906 true,
8907 false,
8908 Some(dir.path().to_path_buf()),
8909 )
8910 .unwrap();
8911
8912 registry.inner.shutdown.store(true, Ordering::SeqCst);
8913 std::thread::sleep(Duration::from_millis(550));
8914
8915 let task = registry.task_for_session(&task_id, "session").unwrap();
8916
8917 let started = Instant::now();
8918 while !task.paths.exit.exists() {
8919 assert!(
8920 started.elapsed() < CHILD_EXIT_LIVENESS_BOUND,
8921 "exit marker should land quickly for `true`"
8922 );
8923 std::thread::sleep(Duration::from_millis(20));
8924 }
8925
8926 let zombie_pid;
8927 {
8928 let mut state = task.state.lock().unwrap();
8929 state.metadata.status = BgTaskStatus::Running;
8930 state.metadata.status_reason = None;
8931 state.metadata.exit_code = None;
8932 state.metadata.finished_at = None;
8933 state.metadata.duration_ms = None;
8934 crate::bash_background::persistence::write_task(&task.paths.json, &state.metadata)
8935 .expect("persist reset Running metadata");
8936 let zombie = spawn_unreaped_zombie();
8937 zombie_pid = zombie.id();
8938 state.runtime = TaskRuntime::Piped(Some(zombie));
8939 }
8940 *task.terminal_at.lock().unwrap() = None;
8941
8942 assert!(
8943 is_zombie(zombie_pid),
8944 "precondition: stand-in child {zombie_pid} must be a zombie before kill"
8945 );
8946
8947 registry
8949 .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
8950 .expect("kill should succeed");
8951
8952 {
8953 let state = task.state.lock().unwrap();
8954 assert!(
8955 matches!(state.runtime, TaskRuntime::Piped(None)),
8956 "child handle must be released after marker-aware kill"
8957 );
8958 assert!(state.metadata.status.is_terminal());
8959 }
8960
8961 assert!(
8962 !is_zombie(zombie_pid),
8963 "issue #91 regression: child {zombie_pid} left as <defunct> zombie \
8964 after a marker-aware kill"
8965 );
8966 }
8967
8968 #[test]
8969 fn cleanup_finished_keeps_running_tasks() {
8970 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
8971 let dir = tempfile::tempdir().unwrap();
8972 let task_id = registry
8973 .spawn(
8974 SpawnPlan::Unsandboxed,
8975 LONG_RUNNING_COMMAND,
8976 "session".to_string(),
8977 dir.path().to_path_buf(),
8978 HashMap::new(),
8979 Some(Duration::from_secs(30)),
8980 dir.path().to_path_buf(),
8981 10,
8982 true,
8983 false,
8984 Some(dir.path().to_path_buf()),
8985 )
8986 .unwrap();
8987
8988 registry.cleanup_finished(Duration::ZERO);
8989
8990 assert!(registry.inner.tasks.lock().unwrap().contains_key(&task_id));
8991 let _ = registry.kill(&task_id, "session");
8992 }
8993
8994 #[cfg(unix)]
8995 #[test]
8996 fn rehydrating_sandboxed_task_never_respawns_persisted_command() {
8997 let project = tempfile::tempdir().unwrap();
8998 let storage = tempfile::tempdir().unwrap();
8999 let sandbox_temp = storage.path().join("sandbox-temp");
9000 fs::create_dir(&sandbox_temp).unwrap();
9001 let launcher_script = project.path().join("sandbox-launch");
9002 let launcher = PathBuf::from("/bin/sh");
9003 fs::write(
9004 &launcher_script,
9005 "while [ \"$#\" -gt 0 ]; do\n if [ \"$1\" = -- ]; then\n shift\n exec \"$@\"\n fi\n shift\ndone\nexit 78\n",
9006 )
9007 .unwrap();
9008 let mut permissions = fs::metadata(&launcher_script).unwrap().permissions();
9009 permissions.set_mode(0o700);
9010 fs::set_permissions(&launcher_script, permissions).unwrap();
9011
9012 let profile = crate::sandbox_profile::SandboxProfile::build(
9013 vec![project.path().to_path_buf()],
9014 Vec::new(),
9015 Vec::new(),
9016 Vec::new(),
9017 Vec::new(),
9018 Vec::new(),
9019 Vec::new(),
9020 sandbox_temp,
9021 )
9022 .unwrap();
9023 let plan = SpawnPlan::launcher_for_test(profile, launcher);
9024 let spawn_marker = project.path().join("spawn-count");
9025 let stop_marker = project.path().join("stop-command");
9026 let quote =
9027 |path: &Path| format!("'{}'", path.display().to_string().replace('\'', "'\\''"));
9028 let command = format!(
9029 "printf 'spawn\\n' >> {}; while [ ! -e {} ]; do sleep 0.05; done",
9030 quote(&spawn_marker),
9031 quote(&stop_marker)
9032 );
9033
9034 let original = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
9035 let task_id = original
9036 .spawn(
9037 plan,
9038 &command,
9039 "sandbox-rehydrate".to_string(),
9040 project.path().to_path_buf(),
9041 HashMap::new(),
9042 Some(Duration::from_secs(30)),
9043 storage.path().to_path_buf(),
9044 10,
9045 true,
9046 false,
9047 Some(project.path().to_path_buf()),
9048 )
9049 .unwrap();
9050 let started = Instant::now();
9051 while !spawn_marker.exists() {
9052 assert!(
9053 started.elapsed() < Duration::from_secs(20),
9054 "original sandboxed task did not start"
9055 );
9056 std::thread::sleep(Duration::from_millis(10));
9057 }
9058 original.detach();
9059
9060 let restarted = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
9061 restarted
9062 .replay_session(storage.path(), "sandbox-rehydrate")
9063 .unwrap();
9064 let replayed = restarted
9065 .status(
9066 &task_id,
9067 "sandbox-rehydrate",
9068 Some(project.path()),
9069 Some(storage.path()),
9070 4096,
9071 )
9072 .expect("rehydrated sandbox task");
9073 assert_eq!(replayed.info.status, BgTaskStatus::Running);
9074 assert!(replayed.sandbox_native);
9075
9076 std::thread::sleep(Duration::from_millis(650));
9077 assert_eq!(
9078 fs::read_to_string(&spawn_marker).unwrap().lines().count(),
9079 1,
9080 "registry replay must observe the persisted process without spawning its command"
9081 );
9082
9083 fs::write(&stop_marker, "stop").unwrap();
9084 let terminal = wait_for_terminal_snapshot(
9085 &restarted,
9086 &task_id,
9087 "sandbox-rehydrate",
9088 project.path(),
9089 storage.path(),
9090 );
9091 assert_eq!(terminal.info.status, BgTaskStatus::Completed);
9092 assert_eq!(
9093 fs::read_to_string(&spawn_marker).unwrap().lines().count(),
9094 1
9095 );
9096 restarted.detach();
9097 }
9098
9099 #[cfg(windows)]
9100 fn wait_for_file(path: &Path) -> String {
9101 let started = Instant::now();
9108 loop {
9109 if let Ok(content) = fs::read_to_string(path) {
9110 if !content.trim().is_empty() {
9111 return content;
9112 }
9113 }
9114 assert!(
9115 started.elapsed() < Duration::from_secs(30),
9116 "timed out waiting for non-empty {}",
9117 path.display()
9118 );
9119 std::thread::sleep(Duration::from_millis(100));
9120 }
9121 }
9122
9123 #[cfg(windows)]
9124 fn spawn_windows_registry_command(
9125 command: &str,
9126 ) -> (BgTaskRegistry, tempfile::TempDir, String) {
9127 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
9128 let dir = tempfile::tempdir().unwrap();
9129 let task_id = registry
9130 .spawn(
9131 SpawnPlan::Unsandboxed,
9132 command,
9133 "session".to_string(),
9134 dir.path().to_path_buf(),
9135 HashMap::new(),
9136 Some(Duration::from_secs(30)),
9137 dir.path().to_path_buf(),
9138 10,
9139 false,
9140 false,
9141 Some(dir.path().to_path_buf()),
9142 )
9143 .unwrap();
9144 (registry, dir, task_id)
9145 }
9146
9147 #[cfg(windows)]
9148 #[test]
9149 fn windows_spawn_writes_exit_marker_for_zero_exit() {
9150 let (registry, _dir, task_id) = spawn_windows_registry_command("cmd /c exit 0");
9151 let exit_path = registry.task_exit_path(&task_id, "session").unwrap();
9152
9153 let content = wait_for_file(&exit_path);
9154
9155 assert_eq!(content.trim(), "0");
9156 }
9157
9158 #[cfg(windows)]
9159 #[test]
9160 fn windows_spawn_writes_exit_marker_for_nonzero_exit() {
9161 let (registry, _dir, task_id) = spawn_windows_registry_command("cmd /c exit 42");
9162 let exit_path = registry.task_exit_path(&task_id, "session").unwrap();
9163
9164 let content = wait_for_file(&exit_path);
9165
9166 assert_eq!(content.trim(), "42");
9167 }
9168
9169 #[cfg(windows)]
9170 #[test]
9171 fn windows_spawn_captures_stdout_to_disk() {
9172 let (registry, _dir, task_id) = spawn_windows_registry_command("cmd /c echo hello");
9173 let task = registry.task_for_session(&task_id, "session").unwrap();
9174 let stdout_path = task.paths.stdout.clone();
9175 let exit_path = task.paths.exit.clone();
9176
9177 let _ = wait_for_file(&exit_path);
9178 let stdout = fs::read_to_string(stdout_path).expect("read stdout");
9179
9180 assert!(stdout.contains("hello"), "stdout was {stdout:?}");
9181 }
9182
9183 #[cfg(windows)]
9184 #[test]
9185 fn windows_spawn_uses_pwsh_when_available() {
9186 let candidates = crate::windows_shell::shell_candidates_with(
9190 |binary| match binary {
9191 "pwsh.exe" => Some(std::path::PathBuf::from(r"C:\pwsh\pwsh.exe")),
9192 "powershell.exe" => Some(std::path::PathBuf::from(r"C:\ps\powershell.exe")),
9193 _ => None,
9194 },
9195 || None,
9196 );
9197 let shell = candidates.first().expect("at least one candidate").clone();
9198 assert_eq!(shell, crate::windows_shell::WindowsShell::Pwsh);
9199 assert_eq!(shell.binary().as_ref(), "pwsh.exe");
9200 }
9201
9202 #[cfg(windows)]
9205 #[test]
9206 fn windows_shell_cmd_wrapper_writes_marker_via_temp_rename() {
9207 let exit_path = Path::new(r"C:\Temp\bash-test.exit");
9208 let script =
9209 crate::windows_shell::WindowsShell::Cmd.wrapper_script("cmd /c exit 42", exit_path);
9210
9211 assert!(
9212 script.contains("set CODE=%ERRORLEVEL%"),
9213 "wrapper must capture the child exit code: {script}"
9214 );
9215 assert!(
9216 script.contains("exit /B %CODE%"),
9217 "wrapper must propagate the child exit code: {script}"
9218 );
9219 assert!(
9224 script.contains("bash-test.exit"),
9225 "wrapper must target the exit marker path: {script}"
9226 );
9227 assert!(
9228 script.contains("move /Y"),
9229 "wrapper must write the marker atomically via temp-file + rename: {script}"
9230 );
9231 }
9232
9233 #[cfg(windows)]
9239 #[test]
9240 fn windows_shell_cmd_bg_command_uses_minimal_cmd_flags() {
9241 use crate::windows_shell::WindowsShell;
9242 let cmd = WindowsShell::Cmd.bg_command("echo wrapped");
9243 let args: Vec<&std::ffi::OsStr> = cmd.get_args().collect();
9244 let args_strs: Vec<&str> = args.iter().filter_map(|a| a.to_str()).collect();
9245 assert_eq!(
9246 args_strs,
9247 vec!["/D", "/S", "/C", "echo wrapped"],
9248 "Cmd::bg_command must prepend /D /S /C"
9249 );
9250 }
9251
9252 #[cfg(windows)]
9255 #[test]
9256 fn windows_shell_pwsh_bg_command_uses_standard_args() {
9257 use crate::windows_shell::WindowsShell;
9258 let cmd = WindowsShell::Pwsh.bg_command("Get-Date");
9259 let args: Vec<&std::ffi::OsStr> = cmd.get_args().collect();
9260 let args_strs: Vec<&str> = args.iter().filter_map(|a| a.to_str()).collect();
9261 assert!(
9262 args_strs.contains(&"-Command"),
9263 "Pwsh::bg_command must use -Command: {args_strs:?}"
9264 );
9265 assert!(
9266 args_strs.contains(&"Get-Date"),
9267 "Pwsh::bg_command must include the user command body"
9268 );
9269 }
9270
9271 fn registry_with_db_and_frames(
9272 storage: &Path,
9273 ) -> (
9274 BgTaskRegistry,
9275 Arc<Mutex<TrackedConnection>>,
9276 Arc<Mutex<Vec<PushFrame>>>,
9277 ) {
9278 let frames = Arc::new(Mutex::new(Vec::new()));
9279 let captured = Arc::clone(&frames);
9280 let sender: crate::context::ProgressSender = Arc::new(Box::new(move |frame| {
9281 captured.lock().unwrap().push(frame);
9282 })
9283 as Box<dyn Fn(PushFrame) + Send + Sync>);
9284 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(Some(sender))));
9285 registry.set_harness(Harness::Opencode);
9286 let conn = crate::db::open(&storage.join("aft.db")).expect("open test DB");
9287 let shared = Arc::new(Mutex::new(conn));
9288 registry.set_db_pool(shared.clone());
9289 (registry, shared, frames)
9290 }
9291
9292 fn pattern_match_frames(frames: &Mutex<Vec<PushFrame>>) -> Vec<BashPatternMatchFrame> {
9293 frames
9294 .lock()
9295 .unwrap()
9296 .iter()
9297 .filter_map(|frame| match frame {
9298 PushFrame::BashPatternMatch(frame) => Some(frame.clone()),
9299 _ => None,
9300 })
9301 .collect()
9302 }
9303
9304 fn install_delivered_terminal_with_pending_watch(
9305 registry: &BgTaskRegistry,
9306 db: &Arc<Mutex<TrackedConnection>>,
9307 storage: &Path,
9308 task_id: &str,
9309 ) -> TaskPaths {
9310 let paths = task_paths(storage, "session", task_id).unwrap();
9311 let mut metadata = PersistedTask::starting(
9312 task_id.to_string(),
9313 "session".to_string(),
9314 "false".to_string(),
9315 storage.to_path_buf(),
9316 Some(storage.to_path_buf()),
9317 None,
9318 true,
9319 true,
9320 );
9321 metadata.mark_terminal(BgTaskStatus::Failed, Some(1), None);
9322 metadata.completion_delivered = true;
9323 write_task(&paths.json, &metadata).unwrap();
9324 {
9325 let conn = db.lock().unwrap();
9326 crate::db::bash_tasks::upsert_bash_task(
9327 &conn,
9328 &metadata.to_bash_task_row("opencode", &paths).unwrap(),
9329 )
9330 .unwrap();
9331 crate::db::bash_watches::upsert_bash_pattern_watch(
9332 &conn,
9333 &BashPatternWatchRow {
9334 harness: "opencode".into(),
9335 session_id: "session".into(),
9336 task_id: task_id.into(),
9337 watch_id: "watch-00000001".into(),
9338 pattern_kind: "substring".into(),
9339 pattern: "(fail)".into(),
9340 once: true,
9341 created_at: 1,
9342 stdout_offset: 756_243,
9343 stderr_offset: 0,
9344 pty_offset: 0,
9345 scanning: false,
9346 pending_match: true,
9347 match_text: Some("(fail)".into()),
9348 match_offset: Some(756_237),
9349 match_context: Some("release output ... (fail)".into()),
9350 },
9351 )
9352 .unwrap();
9353 }
9354 registry
9355 .insert_rehydrated_task(metadata, paths.clone(), true)
9356 .unwrap();
9357 paths
9358 }
9359
9360 #[test]
9365 fn gc_skips_a_task_directory_still_being_created_but_quarantines_an_abandoned_one() {
9366 let dir = tempfile::tempdir().unwrap();
9367 let storage = dir.path();
9368 let registry = BgTaskRegistry::default();
9369 let session_dir = session_tasks_dir(storage, "session");
9370 let young = session_dir.join("bash-0000000000000301");
9371 let abandoned = session_dir.join("bash-0000000000000302");
9372 fs::create_dir_all(&young).unwrap();
9373 fs::create_dir_all(&abandoned).unwrap();
9374 let old = SystemTime::now() - Duration::from_secs(6 * 60);
9375 filetime::set_file_mtime(&abandoned, filetime::FileTime::from_system_time(old)).unwrap();
9376
9377 registry.maybe_gc_persisted(storage).unwrap();
9378
9379 assert!(
9380 young.is_dir(),
9381 "a task directory younger than the grace was quarantined mid-creation"
9382 );
9383 assert!(
9384 !abandoned.is_dir(),
9385 "an empty task directory older than the grace must still be quarantined"
9386 );
9387 let quarantined = fs::read_dir(storage.join("bash-tasks-quarantine"))
9388 .map(|entries| entries.flatten().count())
9389 .unwrap_or(0);
9390 assert_eq!(
9391 quarantined, 1,
9392 "exactly the abandoned layout is quarantined"
9393 );
9394 }
9395
9396 #[test]
9397 fn pending_pattern_match_is_returned_by_drain_contract_until_ack() {
9398 let storage = tempfile::tempdir().unwrap();
9399 let (registry, db, _frames) = registry_with_db_and_frames(storage.path());
9400 let task_id = "bash-0000000000000197";
9401 install_delivered_terminal_with_pending_watch(®istry, &db, storage.path(), task_id);
9402
9403 let first = registry.pending_pattern_matches_for_session("session");
9404 let second = registry.pending_pattern_matches_for_session("session");
9405 assert_eq!(first.len(), 1);
9406 assert_eq!(second.len(), 1);
9407 assert_eq!(first[0].task_id, task_id);
9408 assert_eq!(first[0].watch_id, "watch-00000001");
9409 assert_eq!(registry.unacked_wake_count_for_session(Some("session")), 1);
9410 let stuck =
9411 registry.stuck_pending_watches_for_session("session", Duration::from_secs(10 * 60));
9412 assert_eq!(stuck.len(), 1);
9413 assert_eq!(stuck[0].0, task_id);
9414 assert_eq!(stuck[0].1, "watch-00000001");
9415 assert!(stuck[0].2 >= Duration::from_secs(10 * 60).as_millis() as u64);
9416
9417 assert_eq!(
9418 registry.ack_completions_for_session(Some("session"), &[task_id.to_string()],),
9419 vec![task_id.to_string()]
9420 );
9421 assert!(registry
9422 .pending_pattern_matches_for_session("session")
9423 .is_empty());
9424 assert_eq!(registry.unacked_wake_count_for_session(Some("session")), 0);
9425 }
9426
9427 #[test]
9428 fn terminalized_watch_is_one_shot_across_registry_restart_after_ack() {
9429 let storage = tempfile::tempdir().unwrap();
9430 let (registry, db, _frames) = registry_with_db_and_frames(storage.path());
9431 let task_id = "bash-0000000000000196";
9432 install_delivered_terminal_with_pending_watch(®istry, &db, storage.path(), task_id);
9433 {
9434 let conn = db.lock().unwrap();
9435 let mut row = crate::db::bash_watches::get_bash_pattern_watch(
9436 &conn,
9437 "opencode",
9438 "session",
9439 task_id,
9440 "watch-00000001",
9441 )
9442 .unwrap()
9443 .unwrap();
9444 row.scanning = false;
9445 row.pending_match = true;
9446 row.match_text = Some(WATCH_TARGET_ERASED_TEXT.to_string());
9447 row.match_offset = Some(0);
9448 row.match_context = Some(WATCH_TARGET_ERASED_CONTEXT.to_string());
9449 crate::db::bash_watches::upsert_bash_pattern_watch(&conn, &row).unwrap();
9450 }
9451
9452 let terminalized = registry.pending_pattern_matches_for_session("session");
9453 assert_eq!(terminalized.len(), 1);
9454 assert_eq!(terminalized[0].task_id, task_id);
9455 assert_eq!(terminalized[0].match_text, WATCH_TARGET_ERASED_TEXT);
9456 assert_eq!(
9457 registry.ack_completions_for_session(Some("session"), &[task_id.to_string()]),
9458 vec![task_id.to_string()]
9459 );
9460 drop(registry);
9461
9462 let restarted = BgTaskRegistry::default();
9463 restarted.set_harness(Harness::Opencode);
9464 restarted.set_db_pool(db);
9465 assert!(restarted
9466 .pending_pattern_matches_for_session("session")
9467 .is_empty());
9468 }
9469
9470 #[test]
9471 fn standalone_same_project_multi_session_replay_preserves_earlier_session_delivery() {
9472 let storage = tempfile::tempdir().unwrap();
9473 let project = tempfile::tempdir().unwrap();
9474 let (registry, _db, frames) = registry_with_db_and_frames(storage.path());
9475
9476 registry
9478 .replay_session_for_project(storage.path(), "session-a", project.path())
9479 .unwrap();
9480
9481 let task_watch = registry
9482 .spawn(
9483 SpawnPlan::Unsandboxed,
9484 LONG_RUNNING_COMMAND,
9485 "session-a".to_string(),
9486 project.path().to_path_buf(),
9487 HashMap::new(),
9488 Some(Duration::from_secs(30)),
9489 storage.path().to_path_buf(),
9490 10,
9491 true,
9492 false,
9493 Some(project.path().to_path_buf()),
9494 )
9495 .unwrap();
9496 registry
9497 .register_watch(
9498 task_watch.clone(),
9499 WatchPattern::Substring("READY".into()),
9500 true,
9501 )
9502 .unwrap();
9503
9504 let task_comp = registry
9505 .spawn(
9506 SpawnPlan::Unsandboxed,
9507 LONG_RUNNING_COMMAND,
9508 "session-a".to_string(),
9509 project.path().to_path_buf(),
9510 HashMap::new(),
9511 Some(Duration::from_secs(30)),
9512 storage.path().to_path_buf(),
9513 10,
9514 true,
9515 false,
9516 Some(project.path().to_path_buf()),
9517 )
9518 .unwrap();
9519
9520 registry
9522 .replay_session_for_project(storage.path(), "session-b", project.path())
9523 .unwrap();
9524
9525 let watch_handle = registry.task_for_session(&task_watch, "session-a").unwrap();
9527 std::fs::OpenOptions::new()
9528 .append(true)
9529 .open(&watch_handle.paths.stdout)
9530 .unwrap()
9531 .write_all(b"READY\n")
9532 .unwrap();
9533 registry.scan_task_watch_output(&watch_handle);
9534
9535 let comp_handle = registry.task_for_session(&task_comp, "session-a").unwrap();
9537 fs::write(&comp_handle.paths.exit, "0\n").unwrap();
9538 comp_handle.mark_terminal_now();
9539 comp_handle.state.lock().unwrap().metadata.mark_terminal(
9540 BgTaskStatus::Completed,
9541 Some(0),
9542 None,
9543 );
9544 registry
9545 .post_terminal_transition(&comp_handle, true)
9546 .unwrap();
9547
9548 let deadline = Instant::now() + Duration::from_secs(5);
9552 loop {
9553 let emitted = frames.lock().unwrap().iter().any(
9554 |frame| matches!(frame, PushFrame::BashCompleted(f) if f.task_id == task_comp),
9555 );
9556 if emitted || Instant::now() >= deadline {
9557 break;
9558 }
9559 std::thread::sleep(Duration::from_millis(20));
9560 }
9561
9562 let captured = frames.lock().unwrap();
9563 let pattern_matches = captured
9564 .iter()
9565 .filter_map(|frame| match frame {
9566 PushFrame::BashPatternMatch(frame) => Some(frame),
9567 _ => None,
9568 })
9569 .collect::<Vec<_>>();
9570 assert!(
9571 pattern_matches.iter().any(|f| f.task_id == task_watch
9572 && f.session_id == "session-a"
9573 && f.match_text == "READY"),
9574 "session A pattern match must be emitted after session B replay: {pattern_matches:?}"
9575 );
9576
9577 let completions = captured
9578 .iter()
9579 .filter_map(|frame| match frame {
9580 PushFrame::BashCompleted(frame) => Some(frame),
9581 _ => None,
9582 })
9583 .collect::<Vec<_>>();
9584 assert!(
9585 completions
9586 .iter()
9587 .any(|f| f.task_id == task_comp && f.session_id == "session-a"),
9588 "session A completion must be emitted after session B replay: {completions:?}"
9589 );
9590 }
9591
9592 #[cfg(unix)]
9593 #[test]
9594 fn gc_refuses_to_delete_or_quarantine_a_recorded_live_process() {
9595 let dir = tempfile::tempdir().unwrap();
9596 let storage = dir.path();
9597 let (registry, db, _frames) = registry_with_db_and_frames(storage);
9598 let task_id = "bash-0000000000000198";
9599 let paths = task_paths(storage, "session", task_id).unwrap();
9600 let mut running = PersistedTask::starting(
9601 task_id.to_string(),
9602 "session".to_string(),
9603 "live-process-canary".to_string(),
9604 storage.to_path_buf(),
9605 Some(storage.to_path_buf()),
9606 None,
9607 true,
9608 false,
9609 );
9610 running.mark_running(std::process::id(), std::process::id() as i32);
9614 write_task(&paths.json, &running).unwrap();
9615 fs::write(&paths.stdout, b"").unwrap();
9616 fs::write(&paths.stderr, b"").unwrap();
9617 {
9618 let conn = db.lock().unwrap();
9619 crate::db::bash_tasks::upsert_bash_task(
9620 &conn,
9621 &running.to_bash_task_row("opencode", &paths).unwrap(),
9622 )
9623 .unwrap();
9624 }
9625 let running_json = fs::read(&paths.json).unwrap();
9626 let mut terminal: PersistedTask = serde_json::from_slice(&running_json).unwrap();
9627 terminal.mark_terminal(BgTaskStatus::Completed, Some(0), None);
9628 terminal.completion_delivered = true;
9629 write_task_at(
9630 &resolve_task_layout(&paths.session_dir, task_id).unwrap(),
9631 &terminal,
9632 )
9633 .unwrap();
9634 let old = SystemTime::now()
9635 .checked_sub(Duration::from_secs(25 * 60 * 60))
9636 .unwrap();
9637 filetime::set_file_mtime(&paths.json, filetime::FileTime::from_system_time(old)).unwrap();
9638
9639 assert_eq!(registry.maybe_gc_persisted(storage).unwrap(), 0);
9640 assert!(paths.io_dir.exists(), "GC deleted a live task bundle");
9641
9642 fs::write(&paths.json, b"{corrupt").unwrap();
9643 filetime::set_file_mtime(&paths.json, filetime::FileTime::from_system_time(old)).unwrap();
9644 assert_eq!(registry.maybe_gc_persisted(storage).unwrap(), 0);
9645 assert!(
9646 paths.io_dir.exists(),
9647 "GC quarantined a live task with unreadable metadata"
9648 );
9649
9650 fs::write(&paths.json, running_json).unwrap();
9651 }
9652
9653 #[test]
9654 fn pattern_watch_survives_registry_teardown_and_rehydrate() {
9655 let dir = tempfile::tempdir().unwrap();
9656 let storage = dir.path();
9657 let (registry, _db, frames) = registry_with_db_and_frames(storage);
9658 let task_id = registry
9659 .spawn(
9660 SpawnPlan::Unsandboxed,
9661 LONG_RUNNING_COMMAND,
9662 "session".to_string(),
9663 storage.to_path_buf(),
9664 HashMap::new(),
9665 Some(Duration::from_secs(30)),
9666 storage.to_path_buf(),
9667 10,
9668 true,
9669 false,
9670 Some(storage.to_path_buf()),
9671 )
9672 .unwrap();
9673 registry
9674 .register_watch(
9675 task_id.clone(),
9676 WatchPattern::Substring("READY".into()),
9677 true,
9678 )
9679 .unwrap();
9680 let task = registry.task_for_session(&task_id, "session").unwrap();
9681 registry.clear_task_watch_state(&task_id);
9683 assert_eq!(registry.active_watch_count(&task_id), 0);
9684
9685 std::fs::OpenOptions::new()
9686 .append(true)
9687 .open(&task.paths.stdout)
9688 .unwrap()
9689 .write_all(b"READY\n")
9690 .unwrap();
9691 frames.lock().unwrap().clear();
9692
9693 let (replayed, _db2, replay_frames) = registry_with_db_and_frames(storage);
9694 registry
9696 .inner
9697 .shutdown
9698 .store(true, std::sync::atomic::Ordering::SeqCst);
9699 replayed
9700 .replay_session_inner(storage, "session", None)
9701 .unwrap();
9702
9703 let matches = pattern_match_frames(&replay_frames);
9704 assert!(
9705 matches.iter().any(|frame| {
9706 frame.task_id == task_id
9707 && frame.reason == "pattern_match"
9708 && frame.match_text == "READY"
9709 }),
9710 "rehydrate should deliver gap match: {matches:?}"
9711 );
9712 }
9713
9714 #[test]
9715 fn pattern_watch_gap_match_between_teardown_and_rehydrate_delivers_once() {
9716 let dir = tempfile::tempdir().unwrap();
9717 let storage = dir.path();
9718 let (registry, _db, frames) = registry_with_db_and_frames(storage);
9719 let task_id = registry
9720 .spawn(
9721 SpawnPlan::Unsandboxed,
9722 LONG_RUNNING_COMMAND,
9723 "session".to_string(),
9724 storage.to_path_buf(),
9725 HashMap::new(),
9726 Some(Duration::from_secs(30)),
9727 storage.to_path_buf(),
9728 10,
9729 true,
9730 false,
9731 Some(storage.to_path_buf()),
9732 )
9733 .unwrap();
9734 registry
9735 .register_watch(
9736 task_id.clone(),
9737 WatchPattern::Substring("GAP-HIT".into()),
9738 true,
9739 )
9740 .unwrap();
9741 let task = registry.task_for_session(&task_id, "session").unwrap();
9742 let cursor_before = registry.watch_stream_cursors(&task_id).0;
9743 registry.clear_task_watch_state(&task_id);
9744
9745 std::fs::OpenOptions::new()
9747 .append(true)
9748 .open(&task.paths.stdout)
9749 .unwrap()
9750 .write_all(b"prefix GAP-HIT suffix\n")
9751 .unwrap();
9752 frames.lock().unwrap().clear();
9753
9754 let (replayed, _db2, replay_frames) = registry_with_db_and_frames(storage);
9755 registry
9756 .inner
9757 .shutdown
9758 .store(true, std::sync::atomic::Ordering::SeqCst);
9759 replayed
9760 .replay_session_inner(storage, "session", None)
9761 .unwrap();
9762
9763 let matches: Vec<_> = pattern_match_frames(&replay_frames)
9764 .into_iter()
9765 .filter(|frame| frame.task_id == task_id && frame.match_text.contains("GAP-HIT"))
9766 .collect();
9767 assert_eq!(
9768 matches.len(),
9769 1,
9770 "gap match must deliver exactly once: {matches:?}"
9771 );
9772 assert!(
9773 matches[0].match_offset >= cursor_before,
9774 "match offset should be at/after the persisted cursor ({cursor_before}), got {}",
9775 matches[0].match_offset
9776 );
9777 }
9778
9779 #[test]
9780 fn pattern_watch_acked_match_does_not_redeliver_after_restart() {
9781 let dir = tempfile::tempdir().unwrap();
9782 let storage = dir.path();
9783 let (registry, db, frames) = registry_with_db_and_frames(storage);
9784 let task_id = registry
9785 .spawn(
9786 SpawnPlan::Unsandboxed,
9787 LONG_RUNNING_COMMAND,
9788 "session".to_string(),
9789 storage.to_path_buf(),
9790 HashMap::new(),
9791 Some(Duration::from_secs(30)),
9792 storage.to_path_buf(),
9793 10,
9794 true,
9795 false,
9796 Some(storage.to_path_buf()),
9797 )
9798 .unwrap();
9799 registry
9800 .register_watch(
9801 task_id.clone(),
9802 WatchPattern::Substring("READY".into()),
9803 true,
9804 )
9805 .unwrap();
9806 let task = registry.task_for_session(&task_id, "session").unwrap();
9807 std::fs::OpenOptions::new()
9808 .append(true)
9809 .open(&task.paths.stdout)
9810 .unwrap()
9811 .write_all(b"READY\n")
9812 .unwrap();
9813 registry.scan_task_watch_output(&task);
9814 let delivered = pattern_match_frames(&frames);
9815 assert!(
9816 delivered
9817 .iter()
9818 .any(|frame| frame.task_id == task_id && frame.match_text == "READY"),
9819 "live path should deliver match: {delivered:?}"
9820 );
9821 assert!(registry
9823 .ack_completions_for_session(Some("session"), std::slice::from_ref(&task_id))
9824 .contains(&task_id));
9825 {
9826 let conn = db.lock().unwrap();
9827 let rows = crate::db::bash_watches::list_bash_pattern_watches_for_task(
9828 &conn, "opencode", "session", &task_id,
9829 )
9830 .unwrap();
9831 assert!(
9832 rows.is_empty(),
9833 "acked once-watch rows must be deleted: {rows:?}"
9834 );
9835 }
9836
9837 frames.lock().unwrap().clear();
9838 let (replayed, _db2, replay_frames) = registry_with_db_and_frames(storage);
9839 registry
9840 .inner
9841 .shutdown
9842 .store(true, std::sync::atomic::Ordering::SeqCst);
9843 replayed
9844 .replay_session_inner(storage, "session", None)
9845 .unwrap();
9846 let matches = pattern_match_frames(&replay_frames)
9847 .into_iter()
9848 .filter(|frame| frame.task_id == task_id)
9849 .collect::<Vec<_>>();
9850 assert!(
9851 matches.is_empty(),
9852 "acked match must not re-deliver after restart: {matches:?}"
9853 );
9854 }
9855
9856 #[test]
9857 fn pending_watch_match_redelivers_after_terminal_cleanup_removes_task_bundle() {
9858 let dir = tempfile::tempdir().unwrap();
9859 let storage = dir.path();
9860 let (registry, db, frames) = registry_with_db_and_frames(storage);
9861 let task_id = "bash-aaaaaaaaaaaaaaa1";
9862 let paths = install_delivered_terminal_with_pending_watch(®istry, &db, storage, task_id);
9863 frames.lock().unwrap().clear();
9864
9865 registry.cleanup_finished(Duration::ZERO);
9866 assert!(registry.task(task_id).is_none());
9867 assert!(
9868 !paths.json.exists(),
9869 "cleanup must remove the task metadata bundle"
9870 );
9871
9872 let _ = registry.drain_completions_for_session(Some("session"));
9873 let matches = pattern_match_frames(&frames);
9874 assert!(
9875 matches.iter().any(|frame| {
9876 frame.task_id == task_id
9877 && frame.watch_id == "watch-00000001"
9878 && frame.match_text == "(fail)"
9879 }),
9880 "durable pending match must redeliver without an in-memory task: {matches:?}"
9881 );
9882 assert!(registry
9883 .ack_completions_for_session(Some("session"), &[task_id.to_string()])
9884 .contains(&task_id.to_string()));
9885 let rows = crate::db::bash_watches::list_bash_pattern_watches_for_task(
9886 &db.lock().unwrap(),
9887 "opencode",
9888 "session",
9889 task_id,
9890 )
9891 .unwrap();
9892 assert!(rows.is_empty(), "ack must end durable redelivery");
9893 }
9894
9895 #[test]
9896 fn status_uses_intact_terminal_db_row_after_task_bundle_cleanup() {
9897 let dir = tempfile::tempdir().unwrap();
9898 let storage = dir.path();
9899 let (registry, db, _frames) = registry_with_db_and_frames(storage);
9900 let task_id = "bash-aaaaaaaaaaaaaaa2";
9901 let paths = install_delivered_terminal_with_pending_watch(®istry, &db, storage, task_id);
9902
9903 registry.cleanup_finished(Duration::ZERO);
9904 assert!(registry.task(task_id).is_none());
9905 assert!(
9906 !paths.json.exists(),
9907 "cleanup must remove the task metadata bundle"
9908 );
9909 assert!(
9910 crate::db::bash_tasks::get_bash_task(
9911 &db.lock().unwrap(),
9912 "opencode",
9913 "session",
9914 task_id
9915 )
9916 .unwrap()
9917 .is_some(),
9918 "cleanup must retain the terminal database row"
9919 );
9920
9921 let snapshot = registry
9922 .status(
9923 task_id,
9924 "session",
9925 Some(storage),
9926 Some(storage),
9927 RUNNING_OUTPUT_PREVIEW_BYTES,
9928 )
9929 .expect("intact terminal row must remain visible after artifact cleanup");
9930 assert_eq!(snapshot.info.status, BgTaskStatus::Failed);
9931 assert_eq!(snapshot.exit_code, Some(1));
9932 assert!(snapshot.info.duration_ms.is_some());
9933 }
9934
9935 #[test]
9936 fn pattern_watch_rows_are_deleted_when_task_is_gc_deleted() {
9937 let dir = tempfile::tempdir().unwrap();
9938 let storage = dir.path();
9939 let (registry, db, _frames) = registry_with_db_and_frames(storage);
9940 let task_id = "bash-aaaaaaaaaaaaaaaa";
9941 let paths = task_paths(storage, "session", task_id).unwrap();
9942 let mut metadata = PersistedTask::starting(
9943 task_id.to_string(),
9944 "session".to_string(),
9945 "true".to_string(),
9946 storage.to_path_buf(),
9947 Some(storage.to_path_buf()),
9948 None,
9949 true,
9950 true,
9951 );
9952 metadata.mark_terminal(BgTaskStatus::Completed, Some(0), None);
9953 metadata.completion_delivered = true;
9954 write_task(&paths.json, &metadata).unwrap();
9955 {
9956 let conn = db.lock().unwrap();
9957 crate::db::bash_tasks::upsert_bash_task(
9958 &conn,
9959 &metadata.to_bash_task_row("opencode", &paths).unwrap(),
9960 )
9961 .unwrap();
9962 crate::db::bash_watches::upsert_bash_pattern_watch(
9963 &conn,
9964 &BashPatternWatchRow {
9965 harness: "opencode".into(),
9966 session_id: "session".into(),
9967 task_id: task_id.into(),
9968 watch_id: "watch-00000001".into(),
9969 pattern_kind: "substring".into(),
9970 pattern: "x".into(),
9971 once: true,
9972 created_at: 1,
9973 stdout_offset: 0,
9974 stderr_offset: 0,
9975 pty_offset: 0,
9976 scanning: true,
9977 pending_match: false,
9978 match_text: None,
9979 match_offset: None,
9980 match_context: None,
9981 },
9982 )
9983 .unwrap();
9984 }
9985 let old = SystemTime::now()
9986 .checked_sub(Duration::from_secs(25 * 60 * 60))
9987 .unwrap();
9988 filetime::set_file_mtime(&paths.json, filetime::FileTime::from_system_time(old)).unwrap();
9989
9990 let deleted = registry.maybe_gc_persisted(storage).unwrap();
9991 assert!(
9992 deleted >= 1,
9993 "expected GC to delete the terminal task bundle"
9994 );
9995 let conn = db.lock().unwrap();
9996 let watches = crate::db::bash_watches::list_bash_pattern_watches_for_task(
9997 &conn, "opencode", "session", task_id,
9998 )
9999 .unwrap();
10000 assert!(
10001 watches.is_empty(),
10002 "task-row GC must cascade to every persisted watch"
10003 );
10004 }
10005}