use std::future::Future;
use std::pin::Pin;
use std::sync::{Arc, Weak};
use std::task::{Context, Poll};
use tokio_stream::Stream;
use crate::error::Result;
use crate::group::ProcessGroup;
use crate::pump::{Popped, SharedLines, pump_lines_core};
use crate::result::Outcome;
use crate::sys::pid_gate::{PidGate, force_kill};
use super::RunningProcess;
#[must_use = "a Finished carries the run's outcome; inspect `outcome` or it is silently discarded"]
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub struct Finished {
pub outcome: Outcome,
pub stderr: String,
pub stderr_truncated: bool,
}
impl Finished {
#[doc(hidden)]
pub fn from_parts(outcome: Outcome, stderr: impl Into<String>, stderr_truncated: bool) -> Self {
Finished {
outcome,
stderr: stderr.into(),
stderr_truncated,
}
}
}
impl RunningProcess {
pub fn stdout_lines(&mut self) -> Result<StdoutLines> {
let lines = self.drain_stdout_lines()?;
self.arm_stream_deadline();
Ok(lines)
}
pub(super) fn drain_stdout_lines(&mut self) -> Result<StdoutLines> {
self.ensure_stdout_streamable()?;
debug_assert!(
self.stdout_sink.is_none(),
"ensure_stdout_streamable rejects a previously consumed stdout stream"
);
self.ensure_stderr_drain();
let stdout_sink = SharedLines::new(&self.buffer);
match self.backend.take_stdout_reader() {
Some(pipe) => {
self.stdout_pump = Some(tokio::spawn(pump_lines_core(
pipe,
self.stdout_config.clone(),
stdout_sink.clone(),
)));
}
None => stdout_sink.close_now(),
}
self.stdout_sink = Some(stdout_sink.clone());
Ok(StdoutLines {
sink: stdout_sink,
wait: None,
})
}
fn ensure_stderr_drain(&mut self) {
if self.stderr_sink.is_none() {
let stderr_sink = SharedLines::new(&self.buffer);
if let Some(pipe) = self.backend.take_stderr_reader() {
self.stderr_pump = Some(tokio::spawn(pump_lines_core(
pipe,
self.stderr_config.clone(),
stderr_sink.clone(),
)));
}
self.stderr_sink = Some(stderr_sink);
}
}
pub(super) fn ensure_background_drains(&mut self) {
self.ensure_stderr_drain();
let _ = self.drain_stdout_lines();
}
fn arm_stream_deadline(&mut self) {
if self.deadline_task.is_none()
&& let Some(limit) = self.timeout
{
let own_group = self.backend.own_group().map(Arc::downgrade);
let pid = self.pid;
if own_group.is_some() || pid.is_some() {
let grace = self.timeout_grace;
let signal = self.timeout_signal;
let started = self.deadline_anchor;
let timeout_state = self.timeout_state.clone();
let gate = self.pid_gate.clone();
self.deadline_task = Some(tokio::spawn(async move {
if !super::deadline::wait_deadline_and_claim(started, limit, &timeout_state)
.await
{
return; }
if gate.is_retired() {
return;
}
match own_group {
Some(group) => match grace {
Some(grace) => match group.upgrade() {
Some(group) => {
let _ = group.graceful_terminate(grace, signal).await;
}
None => force_kill(&gate), },
None => kill_via_weak(&group, &gate),
},
None => match grace {
Some(grace) => spawn_graceful_kill_and_reap(gate, grace, signal),
None => force_kill(&gate),
},
}
}));
}
}
self.arm_scripted_deadline();
}
pub async fn finish(mut self) -> Result<Finished> {
let mut stdout_discarded = false;
if let Some(pipe) = self.backend.take_stdout_reader() {
let sink = SharedLines::new(&super::discard_sink_policy());
self.stdout_pump = Some(tokio::spawn(pump_lines_core(
pipe,
self.stdout_config.clone(),
sink.clone(),
)));
self.stdout_sink = Some(sink);
stdout_discarded = true;
}
if self.stderr_pump.is_none()
&& let Some(pipe) = self.backend.take_stderr_reader()
{
let sink = SharedLines::new(&self.buffer);
self.stderr_pump = Some(tokio::spawn(pump_lines_core(
pipe,
self.stderr_config.clone(),
sink.clone(),
)));
self.stderr_sink = Some(sink);
}
let raw_outcome = self.drive_to_exit().await?;
self.observe_stdin_task().await;
let pumps: Vec<_> = [self.stdout_pump.take(), self.stderr_pump.take()]
.into_iter()
.flatten()
.collect();
super::join_pumps(pumps).await;
self.finalize_stdin_task().await;
let outcome = self.checked_outcome(raw_outcome)?;
let stdout_check = (!stdout_discarded)
.then_some(self.stdout_sink.as_ref())
.flatten();
for sink in [stdout_check, self.stderr_sink.as_ref()]
.into_iter()
.flatten()
{
if sink.overflowed() {
return Err(crate::ErrorReason::OutputTooLarge {
program: self.program.clone(),
max_lines: self.buffer.max_lines,
max_bytes: self.buffer.max_bytes,
total_lines: sink.count(),
total_bytes: sink.seen_bytes(),
}
.into());
}
}
for sink in [self.stdout_sink.as_ref(), self.stderr_sink.as_ref()]
.into_iter()
.flatten()
{
if let Some(source) = sink.take_read_error() {
return Err(crate::Error::io(source));
}
}
let stderr_truncated = self.stderr_sink.as_ref().is_some_and(|s| s.dropped() > 0);
let stderr = if self.merged_events_stream {
String::new()
} else {
self.stderr_sink
.as_ref()
.map(|sink| sink.drain().join("\n"))
.unwrap_or_default()
};
Ok(Finished {
outcome,
stderr,
stderr_truncated,
})
}
pub fn events(&mut self) -> Result<ProcessEvents> {
self.ensure_stdout_streamable()?;
debug_assert!(
self.stdout_sink.is_none(),
"ensure_stdout_streamable rejects a previously consumed stdout stream"
);
debug_assert!(
self.stderr_sink.is_none(),
"a public output stream consumes stdout before it can arm stderr"
);
let stdout_sink = SharedLines::new(&self.buffer);
match self.backend.take_stdout_reader() {
Some(pipe) => {
self.stdout_pump = Some(tokio::spawn(pump_lines_core(
pipe,
self.stdout_config.clone(),
stdout_sink.clone(),
)));
}
None => stdout_sink.close_now(),
}
self.stdout_sink = Some(stdout_sink.clone());
let stderr_sink = SharedLines::new(&self.buffer);
if let Some(pipe) = self.backend.take_stderr_reader() {
self.stderr_pump = Some(tokio::spawn(pump_lines_core(
pipe,
self.stderr_config.clone(),
stderr_sink.clone(),
)));
} else {
stderr_sink.close_now();
}
self.stderr_sink = Some(stderr_sink.clone());
self.arm_stream_deadline();
let (exit_tx, exit_rx) = tokio::sync::oneshot::channel();
self.exit_event_tx = Some(exit_tx);
self.merged_events_stream = true;
Ok(ProcessEvents {
stdout_sink,
stderr_sink,
stdout_wait: None,
stderr_wait: None,
stdout_done: false,
stderr_done: false,
prefer_stdout: true,
start_pid: self.pid,
started_emitted: false,
exit_rx: Some(exit_rx),
})
}
}
pub(super) fn kill_via_weak(group: &Weak<ProcessGroup>, gate: &PidGate) {
if let Some(group) = group.upgrade() {
let _ = group.kill_all();
}
force_kill(gate);
}
pub(crate) async fn graceful_kill_pid(gate: Arc<PidGate>, grace: std::time::Duration, signal: i32) {
#[cfg(unix)]
{
let target = crate::sys::graceful::UnixChild::new(gate);
crate::sys::graceful::run_pid(&target, signal, grace).await;
}
#[cfg(not(unix))]
{
let _ = (grace, signal);
force_kill(&gate);
}
}
fn spawn_graceful_kill_and_reap(gate: Arc<PidGate>, grace: std::time::Duration, signal: i32) {
drop(tokio::spawn(graceful_kill_pid(gate, grace, signal)));
}
#[cfg(unix)]
pub(super) fn signal_direct_child(pid: Option<u32>, signal: i32) {
let Some(pid) = pid else { return };
unsafe {
libc::kill(pid as i32, signal);
}
}
pub struct StdoutLines {
sink: Arc<SharedLines>,
wait: Option<Pin<Box<dyn Future<Output = ()> + Send>>>,
}
impl std::fmt::Debug for StdoutLines {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("StdoutLines").finish_non_exhaustive()
}
}
impl Stream for StdoutLines {
type Item = String;
fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<String>> {
let this = self.get_mut();
loop {
match this.sink.try_pop() {
Popped::Line(line) => {
this.wait = None;
return Poll::Ready(Some(line));
}
Popped::Closed => return Poll::Ready(None),
Popped::Empty => {
if this.wait.is_none() {
this.wait = Some(Box::pin(this.sink.clone().changed()));
}
match this.wait.as_mut().expect("just set").as_mut().poll(cx) {
Poll::Ready(()) => {
this.wait = None;
continue;
}
Poll::Pending => return Poll::Pending,
}
}
}
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub enum ProcessEvent {
Started {
pid: Option<u32>,
},
Stdout(OutputLine),
Stderr(OutputLine),
Exited(Outcome),
}
impl ProcessEvent {
pub fn name(&self) -> &'static str {
match self {
ProcessEvent::Started { .. } => "started",
ProcessEvent::Stdout(_) => "stdout",
ProcessEvent::Stderr(_) => "stderr",
ProcessEvent::Exited(_) => "exited",
}
}
pub fn text(&self) -> Option<&str> {
match self {
ProcessEvent::Stdout(line) | ProcessEvent::Stderr(line) => Some(line.text()),
ProcessEvent::Started { .. } | ProcessEvent::Exited(_) => None,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub struct OutputLine {
text: String,
}
impl OutputLine {
pub fn text(&self) -> &str {
&self.text
}
#[must_use]
pub fn into_text(self) -> String {
self.text
}
}
pub struct ProcessEvents {
stdout_sink: Arc<SharedLines>,
stderr_sink: Arc<SharedLines>,
stdout_wait: Option<Pin<Box<dyn Future<Output = ()> + Send>>>,
stderr_wait: Option<Pin<Box<dyn Future<Output = ()> + Send>>>,
stdout_done: bool,
stderr_done: bool,
prefer_stdout: bool,
start_pid: Option<u32>,
started_emitted: bool,
exit_rx: Option<tokio::sync::oneshot::Receiver<Outcome>>,
}
impl std::fmt::Debug for ProcessEvents {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("ProcessEvents").finish_non_exhaustive()
}
}
impl Stream for ProcessEvents {
type Item = ProcessEvent;
fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<ProcessEvent>> {
let this = self.get_mut();
if !this.started_emitted {
this.started_emitted = true;
return Poll::Ready(Some(ProcessEvent::Started {
pid: this.start_pid,
}));
}
loop {
for stdout_turn in [this.prefer_stdout, !this.prefer_stdout] {
if stdout_turn && !this.stdout_done {
match this.stdout_sink.try_pop() {
Popped::Line(line) => {
this.stdout_wait = None;
this.prefer_stdout = false; return Poll::Ready(Some(ProcessEvent::Stdout(OutputLine {
text: line,
})));
}
Popped::Closed => {
this.stdout_done = true;
this.stdout_wait = None;
}
Popped::Empty => {}
}
} else if !stdout_turn && !this.stderr_done {
match this.stderr_sink.try_pop() {
Popped::Line(line) => {
this.stderr_wait = None;
this.prefer_stdout = true;
return Poll::Ready(Some(ProcessEvent::Stderr(OutputLine {
text: line,
})));
}
Popped::Closed => {
this.stderr_done = true;
this.stderr_wait = None;
}
Popped::Empty => {}
}
}
}
if this.stdout_done && this.stderr_done {
return match this.exit_rx.as_mut() {
Some(rx) => match Pin::new(rx).poll(cx) {
Poll::Ready(Ok(outcome)) => {
this.exit_rx = None;
Poll::Ready(Some(ProcessEvent::Exited(outcome)))
}
Poll::Ready(Err(_)) => {
this.exit_rx = None;
Poll::Ready(None)
}
Poll::Pending => Poll::Pending,
},
None => Poll::Ready(None),
};
}
let mut any_ready = false;
if !this.stdout_done {
if this.stdout_wait.is_none() {
this.stdout_wait = Some(Box::pin(this.stdout_sink.clone().changed()));
}
if this
.stdout_wait
.as_mut()
.expect("just set")
.as_mut()
.poll(cx)
.is_ready()
{
this.stdout_wait = None;
any_ready = true;
}
}
if !this.stderr_done {
if this.stderr_wait.is_none() {
this.stderr_wait = Some(Box::pin(this.stderr_sink.clone().changed()));
}
if this
.stderr_wait
.as_mut()
.expect("just set")
.as_mut()
.poll(cx)
.is_ready()
{
this.stderr_wait = None;
any_ready = true;
}
}
if any_ready {
continue;
}
return Poll::Pending;
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::buffer::OutputBufferPolicy;
use tokio_stream::StreamExt;
#[tokio::test]
async fn finish_surfaces_a_recorded_stream_read_error_as_io() {
use crate::command::Command;
use crate::doubles::{Reply, ScriptedRunner};
use crate::runner::ProcessRunner;
let mut run = ScriptedRunner::new()
.fallback(Reply::ok(""))
.start(&Command::new("tool"))
.await
.expect("scripted start");
drop(run.stdout_lines().expect("stdout_lines"));
run.stdout_sink
.as_ref()
.expect("streaming installed the stdout sink")
.set_read_error(std::io::Error::other("stream read boom"));
match run.finish().await.map_err(|e| e.into_reason()) {
Err(crate::ErrorReason::Io(e)) => assert_eq!(e.to_string(), "stream read boom"),
other => panic!("expected Err(Io) for an incomplete streamed capture, got {other:?}"),
}
}
fn line_only_events(
stdout_sink: Arc<SharedLines>,
stderr_sink: Arc<SharedLines>,
) -> ProcessEvents {
ProcessEvents {
stdout_sink,
stderr_sink,
stdout_wait: None,
stderr_wait: None,
stdout_done: false,
stderr_done: false,
prefer_stdout: true,
start_pid: None,
started_emitted: true,
exit_rx: None,
}
}
#[tokio::test]
async fn events_interleave_fairly_between_two_ready_streams() {
let policy = OutputBufferPolicy::unbounded();
let stdout_sink = SharedLines::new(&policy);
let stderr_sink = SharedLines::new(&policy);
for line in ["o1", "o2", "o3"] {
stdout_sink.push(line.to_owned());
}
for line in ["e1", "e2", "e3"] {
stderr_sink.push(line.to_owned());
}
stdout_sink.close_now();
stderr_sink.close_now();
let mut events = line_only_events(stdout_sink, stderr_sink);
let mut seq = Vec::new();
while let Some(ev) = events.next().await {
seq.push(match ev {
ProcessEvent::Stdout(l) => format!("O:{}", l.text()),
ProcessEvent::Stderr(l) => format!("E:{}", l.text()),
other => panic!("unexpected event: {other:?}"),
});
}
assert_eq!(
seq,
["O:o1", "E:e1", "O:o2", "E:e2", "O:o3", "E:e3"],
"merged stream must interleave, not drain stdout first"
);
}
#[tokio::test]
async fn process_event_carries_an_output_line_with_a_text_accessor() {
let policy = OutputBufferPolicy::unbounded();
let stdout_sink = SharedLines::new(&policy);
let stderr_sink = SharedLines::new(&policy);
stdout_sink.push("out".to_owned());
stderr_sink.push("err".to_owned());
stdout_sink.close_now();
stderr_sink.close_now();
let mut events = line_only_events(stdout_sink, stderr_sink);
let first = events.next().await.expect("a stdout event");
assert!(
matches!(&first, ProcessEvent::Stdout(line) if line.text() == "out"),
"stdout event carries an OutputLine: {first:?}"
);
assert_eq!(first.text(), Some("out"), "text() reads the line");
let second = events.next().await.expect("a stderr event");
assert!(matches!(&second, ProcessEvent::Stderr(line) if line.text() == "err"));
assert_eq!(second.text(), Some("err"));
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn stdout_lines_loses_no_line_under_a_parking_consumer() {
const N: usize = 5_000;
let sink = SharedLines::new(&OutputBufferPolicy::unbounded());
let producer = {
let sink = sink.clone();
tokio::spawn(async move {
for i in 0..N {
sink.push(i.to_string());
if i % 7 == 0 {
tokio::task::yield_now().await;
}
}
sink.close_now();
})
};
let mut lines = StdoutLines { sink, wait: None };
let consume = async {
let mut seen = 0usize;
while let Some(line) = lines.next().await {
assert_eq!(line, seen.to_string(), "lines must arrive in push order");
seen += 1;
}
seen
};
let seen = tokio::time::timeout(std::time::Duration::from_secs(30), consume)
.await
.expect("consumer hung — possible lost wakeup");
producer.await.expect("producer task");
assert_eq!(seen, N, "every pushed line must be received");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 3)]
async fn events_lose_no_line_under_two_racing_producers() {
const N: usize = 3_000;
let stdout_sink = SharedLines::new(&OutputBufferPolicy::unbounded());
let stderr_sink = SharedLines::new(&OutputBufferPolicy::unbounded());
let feed = |sink: Arc<SharedLines>| {
tokio::spawn(async move {
for i in 0..N {
sink.push(i.to_string());
if i % 5 == 0 {
tokio::task::yield_now().await;
}
}
sink.close_now();
})
};
let p_out = feed(stdout_sink.clone());
let p_err = feed(stderr_sink.clone());
let mut events = line_only_events(stdout_sink, stderr_sink);
let consume = async {
let (mut out, mut err) = (0usize, 0usize);
while let Some(ev) = events.next().await {
match ev {
ProcessEvent::Stdout(_) => out += 1,
ProcessEvent::Stderr(_) => err += 1,
_ => {}
}
}
(out, err)
};
let (out, err) = tokio::time::timeout(std::time::Duration::from_secs(30), consume)
.await
.expect("consumer hung — possible lost wakeup");
p_out.await.expect("stdout producer");
p_err.await.expect("stderr producer");
assert_eq!(out, N, "every stdout line received");
assert_eq!(err, N, "every stderr line received");
}
#[tokio::test]
async fn events_stream_emits_started_lines_and_exited() {
use crate::command::Command;
use crate::doubles::{Reply, ScriptedRunner};
use crate::runner::ProcessRunner;
let mut run = ScriptedRunner::new()
.fallback(Reply::ok("out-1\nout-2").with_stderr("err-1"))
.start(&Command::new("tool"))
.await
.expect("scripted start");
let mut events = run.events().expect("events");
let collect = async {
let mut names = Vec::new();
let mut outcome = None;
while let Some(ev) = events.next().await {
names.push(ev.name());
if let ProcessEvent::Exited(o) = ev {
outcome = Some(o);
}
}
(names, outcome)
};
let ((names, outcome), finished) = tokio::join!(collect, run.finish());
let finished = finished.expect("finish");
assert_eq!(names.first(), Some(&"started"), "Started leads the stream");
assert_eq!(names.last(), Some(&"exited"), "Exited ends the stream");
assert_eq!(
names.iter().filter(|n| **n == "stdout").count(),
2,
"both stdout lines arrive as events"
);
assert_eq!(
names.iter().filter(|n| **n == "stderr").count(),
1,
"the stderr line arrives as an event"
);
assert_eq!(
outcome,
Some(Outcome::Exited(0)),
"Exited carries the run's outcome"
);
assert_eq!(finished.outcome, Outcome::Exited(0));
assert!(
finished.stderr.is_empty(),
"stderr was delivered as events, so Finished::stderr is empty"
);
}
#[tokio::test]
async fn events_stream_over_a_merged_single_stream() {
use crate::command::Command;
use crate::doubles::{Reply, ScriptedRunner};
use crate::runner::ProcessRunner;
let mut run = ScriptedRunner::new()
.fallback(Reply::ok("merged-1\nmerged-2\nmerged-3"))
.start(&Command::new("tool"))
.await
.expect("scripted start");
let mut events = run.events().expect("events");
let collect = async {
let mut seq = Vec::new();
while let Some(ev) = events.next().await {
match ev {
ProcessEvent::Started { pid } => seq.push(format!("started:{pid:?}")),
ProcessEvent::Stdout(l) => seq.push(format!("out:{}", l.text())),
ProcessEvent::Stderr(l) => seq.push(format!("err:{}", l.text())),
ProcessEvent::Exited(o) => seq.push(format!("exited:{}", o.name())),
}
}
seq
};
let (seq, finished) = tokio::join!(collect, run.finish());
let _ = finished.expect("finish");
assert_eq!(
seq,
[
"started:None",
"out:merged-1",
"out:merged-2",
"out:merged-3",
"exited:exited",
],
"Started → stdout lines → Exited, with no stderr events on a merged stream"
);
}
#[test]
fn finished_from_parts_round_trips_every_field() {
let original = Finished::from_parts(Outcome::Signalled(Some(9)), "boom", true);
assert_eq!(original.outcome, Outcome::Signalled(Some(9)));
assert_eq!(original.stderr, "boom");
assert!(original.stderr_truncated);
let rebuilt = Finished::from_parts(
original.outcome,
original.stderr.clone(),
original.stderr_truncated,
);
assert_eq!(original, rebuilt);
}
}