mod streams;
use std::{
error::Error as StdError, ffi::OsString, future::IntoFuture, process::ExitStatus,
string::FromUtf8Error, time::Duration,
};
use indicatif::ProgressBar;
use thiserror::Error;
use tokio::process::Command;
use tokio_process_tools::{
ConsumerError, DEFAULT_MAX_BUFFERED_CHUNKS, DEFAULT_READ_CHUNK_SIZE, GracefulShutdown,
OutputStream, Process, ProcessHandle, SpawnError, TerminationError,
WaitForCompletionOrTerminateResult, WaitOrTerminateError,
};
use tokio_util::sync::CancellationToken;
use self::streams::OutputConsumers;
use crate::{
error::Error,
logs::{CommandLog, CommandLogs, LogSession},
};
#[derive(Debug, Error)]
#[error("`{command}` failed")]
pub(crate) struct CommandError<E>
where
E: StdError + 'static,
{
command: String,
#[source]
source: E,
}
impl<E> CommandError<E>
where
E: StdError + 'static,
{
fn new(command: String, source: E) -> Self {
Self { command, source }
}
}
#[derive(Debug, Error)]
pub(crate) enum ProcessError {
#[error("failed to run command")]
Spawn(#[source] CommandError<SpawnError>),
#[error("failed to wait for command")]
Wait(#[source] CommandError<WaitOrTerminateError>),
#[error("failed to terminate command")]
Terminate(#[source] CommandError<TerminationError>),
#[error("failed to read command output")]
OutputRead(#[source] CommandError<ConsumerError>),
#[error("output from `{command}` did not close after {seconds} seconds")]
OutputDrainTimedOut {
command: String,
seconds: u64,
},
#[error("`{command}` exited with {status}: {stderr}")]
Failed {
command: String,
status: ExitStatus,
stderr: String,
},
#[error("`{command}` timed out after {seconds} seconds")]
TimedOut {
command: String,
seconds: u64,
},
#[error("`{command}` was interrupted")]
Interrupted {
command: String,
},
#[error("`{command}` returned non-UTF-8 output")]
Output {
command: String,
#[source]
source: FromUtf8Error,
},
}
const ERROR_OUTPUT_LINES: usize = 20;
const MAX_COMMAND_RUNTIME: Duration = Duration::from_secs(365 * 24 * 60 * 60);
const TERMINATION_GRACE: Duration = Duration::from_secs(2);
pub(crate) struct ProcessContext<'a> {
logs: &'a LogSession,
cancellation: CancellationToken,
}
impl<'a> ProcessContext<'a> {
pub(crate) fn new(logs: &'a LogSession, cancellation: CancellationToken) -> Self {
Self { logs, cancellation }
}
pub(crate) fn recovery(&self) -> Self {
Self {
logs: self.logs,
cancellation: CancellationToken::new(),
}
}
}
enum Completion {
Exited(ExitStatus),
TimedOut,
Interrupted,
}
fn graceful_shutdown(interrupted: bool) -> GracefulShutdown {
if interrupted {
GracefulShutdown::builder()
.unix_sigint(TERMINATION_GRACE)
.windows_ctrl_break(TERMINATION_GRACE)
.build()
} else {
GracefulShutdown::builder()
.unix_sigterm(TERMINATION_GRACE)
.windows_ctrl_break(TERMINATION_GRACE)
.build()
}
}
async fn wait_for_process<Stdout, Stderr>(
process: &mut ProcessHandle<Stdout, Stderr>,
consumers: &OutputConsumers,
progress: &ProgressBar,
cancellation: &CancellationToken,
limit: Duration,
command: &str,
) -> Result<Completion, ProcessError>
where
Stdout: OutputStream + Send,
Stderr: OutputStream + Send,
{
let mut refresh = consumers.refresh_interval();
let wait_result = {
let wait = process
.wait_for_completion(limit)
.or_terminate(graceful_shutdown(false))
.into_future();
tokio::pin!(wait);
loop {
tokio::select! {
result = &mut wait => break Some(result),
() = cancellation.cancelled() => break None,
_ = refresh.tick() => consumers.refresh(progress),
}
}
};
match wait_result {
Some(Ok(WaitForCompletionOrTerminateResult::Completed(status))) => {
Ok(Completion::Exited(status))
}
Some(Ok(WaitForCompletionOrTerminateResult::TerminatedAfterTimeout { .. })) => {
Ok(Completion::TimedOut)
}
Some(Err(source)) => {
process.must_not_be_terminated();
Err(ProcessError::Wait(CommandError::new(
command.to_owned(),
source,
)))
}
None => match process.terminate(graceful_shutdown(true)).await {
Ok(_status) => Ok(Completion::Interrupted),
Err(source) => {
process.must_not_be_terminated();
Err(ProcessError::Terminate(CommandError::new(
command.to_owned(),
source,
)))
}
},
}
}
fn error_output(stderr: &[u8]) -> String {
let value = String::from_utf8_lossy(stderr);
let mut lines = value
.lines()
.rev()
.take(ERROR_OUTPUT_LINES)
.collect::<Vec<_>>();
lines.reverse();
let summary = lines.join("\n");
if summary.is_empty() {
"no standard error output".to_owned()
} else {
summary
}
}
fn fail_command<T>(log: CommandLog<'_>, error: Error) -> Result<T, Error> {
log.fail(&error.to_string());
Err(error)
}
pub(crate) struct Invocation<'a> {
context: &'a ProcessContext<'a>,
label: String,
program: OsString,
arguments: Vec<OsString>,
environment: Vec<(OsString, OsString)>,
timeout: Option<Duration>,
progress: &'a ProgressBar,
}
impl<'a> Invocation<'a> {
pub(crate) fn new(
context: &'a ProcessContext<'a>,
label: impl Into<String>,
program: impl Into<OsString>,
progress: &'a ProgressBar,
) -> Self {
Self {
context,
label: label.into(),
program: program.into(),
arguments: Vec::new(),
environment: Vec::new(),
timeout: None,
progress,
}
}
#[must_use]
pub(crate) fn argument(mut self, argument: impl Into<OsString>) -> Self {
self.arguments.push(argument.into());
self
}
#[must_use]
pub(crate) fn arguments(
mut self,
arguments: impl IntoIterator<Item = impl Into<OsString>>,
) -> Self {
self.arguments.extend(arguments.into_iter().map(Into::into));
self
}
#[must_use]
pub(crate) fn environment(
mut self,
name: impl Into<OsString>,
value: impl Into<OsString>,
) -> Self {
self.environment.push((name.into(), value.into()));
self
}
#[must_use]
pub(crate) fn timeout(mut self, timeout: Duration) -> Self {
self.timeout = Some(timeout);
self
}
pub(crate) async fn status(self) -> Result<(), Error> {
self.execute(false, |_output| Ok(())).await
}
pub(crate) async fn output(self) -> Result<String, Error> {
self.execute(true, String::from_utf8).await
}
async fn execute<T>(
self,
capture_stdout: bool,
consume: impl FnOnce(Vec<u8>) -> Result<T, FromUtf8Error>,
) -> Result<T, Error> {
let CommandLogs {
lifecycle: command_log,
output: output_logs,
} = self
.context
.logs
.command(&self.label, &self.program, &self.arguments)
.map_err(Error::Log)?;
let mut command = Command::new(&self.program);
command
.args(&self.arguments)
.envs(self.environment.iter().map(|(name, value)| (name, value)));
let command_display = format!("{:?}", command.as_std());
if self.context.cancellation.is_cancelled() {
return fail_command(
command_log,
Error::Process(ProcessError::Interrupted {
command: command_display,
}),
);
}
let mut process = match Process::new(command)
.name(self.label.clone())
.stdout_and_stderr(|stream| {
stream
.single_subscriber()
.reliable_with_backpressure()
.replay_all()
.read_chunk_size(DEFAULT_READ_CHUNK_SIZE)
.max_buffered_chunks(DEFAULT_MAX_BUFFERED_CHUNKS)
})
.spawn()
{
Ok(process) => process,
Err(source) => {
return fail_command(
command_log,
Error::Process(ProcessError::Spawn(CommandError::new(
command_display,
source,
))),
);
}
};
let consumers = OutputConsumers::attach(
process.stdout(),
process.stderr(),
output_logs,
capture_stdout,
);
process.seal_output_replay();
let limit = self.timeout.unwrap_or(MAX_COMMAND_RUNTIME);
let completion = match wait_for_process(
&mut process,
&consumers,
self.progress,
&self.context.cancellation,
limit,
&command_display,
)
.await
{
Ok(completion) => completion,
Err(error) => return fail_command(command_log, Error::Process(error)),
};
let output = match consumers.finish(&command_display, self.progress).await {
Ok(output) => output,
Err(error) => return fail_command(command_log, error),
};
let status = match completion {
Completion::TimedOut => {
return fail_command(
command_log,
Error::Process(ProcessError::TimedOut {
command: command_display,
seconds: limit.as_secs(),
}),
);
}
Completion::Interrupted => {
return fail_command(
command_log,
Error::Process(ProcessError::Interrupted {
command: command_display,
}),
);
}
Completion::Exited(status) => status,
};
if !status.success() {
command_log.complete(status);
return Err(Error::Process(ProcessError::Failed {
command: command_display,
status,
stderr: error_output(&output.stderr),
}));
}
match consume(output.stdout) {
Ok(output) => {
command_log.complete(status);
Ok(output)
}
Err(source) => fail_command(
command_log,
Error::Process(ProcessError::Output {
command: command_display,
source,
}),
),
}
}
}
#[cfg(test)]
mod tests {
use std::{fs, time::Duration};
use indicatif::ProgressBar;
use tempfile::tempdir;
use tokio_util::sync::CancellationToken;
use super::{Invocation, ProcessContext, ProcessError};
use crate::{
error::Error,
logs::{LogMode, LogSession, with_test_log_session},
};
#[tokio::test]
async fn commands_report_timeouts_and_interruption() {
let logs = LogSession::start(LogMode::Never, None).expect("disabled logs should start");
let progress = ProgressBar::hidden();
let timeout_cancellation = CancellationToken::new();
let timeout_context = ProcessContext::new(&logs, timeout_cancellation);
let timeout = Invocation::new(&timeout_context, "timeout", "sh", &progress)
.arguments(["-c", "sleep 10"])
.timeout(Duration::from_millis(20))
.status()
.await;
assert!(matches!(
timeout,
Err(Error::Process(ProcessError::TimedOut { .. }))
));
let cancellation = CancellationToken::new();
let interrupted_context = ProcessContext::new(&logs, cancellation.clone());
let interrupt = async move {
tokio::time::sleep(Duration::from_millis(20)).await;
cancellation.cancel();
};
let command = Invocation::new(&interrupted_context, "interrupted", "sh", &progress)
.arguments(["-c", "sleep 10"])
.status();
let ((), interrupted) = tokio::join!(interrupt, command);
assert!(matches!(
interrupted,
Err(Error::Process(ProcessError::Interrupted { .. }))
));
let recovery_context = interrupted_context.recovery();
Invocation::new(&recovery_context, "recovery", "true", &progress)
.status()
.await
.expect("recovery command should ignore prior cancellation");
drop((timeout_context, interrupted_context, recovery_context));
logs.finish(Some("timed out or interrupted"))
.expect("disabled logs should finish");
}
#[test]
fn invalid_command_output_is_reported() {
let base = tempdir().expect("temporary directory should be created");
let directory = with_test_log_session(LogMode::Always, base.path(), |logs| {
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("Tokio runtime should start");
runtime.block_on(async {
let progress = ProgressBar::hidden();
let context = ProcessContext::new(&logs, CancellationToken::new());
Invocation::new(&context, "ignored-output", "sh", &progress)
.arguments(["-c", "printf '\\377'"])
.status()
.await
.expect("discarded output should not require UTF-8");
let result = Invocation::new(&context, "invalid-output", "sh", &progress)
.arguments(["-c", "printf '\\377'"])
.output()
.await;
let error = result.expect_err("non-UTF-8 output should fail");
assert!(matches!(error, Error::Process(ProcessError::Output { .. })));
});
logs.finish(Some("invalid output"))
.expect("enabled logs should finish")
.expect("always logs should be retained")
});
assert_eq!(
fs::read(directory.join("001.stdout")).expect("ignored output log should be readable"),
[0xff]
);
assert_eq!(
fs::read(directory.join("002.stdout")).expect("invalid output log should be readable"),
[0xff]
);
}
}