use std::fmt::Write as _;
use std::io::{self, Read};
use std::path::PathBuf;
use std::process::{Command, Stdio};
use std::sync::mpsc;
use std::thread;
use std::time::{Duration, Instant};
use crate::domain::{CommitOwnership, MemberId};
use crate::error::{Error, RecoveryError, RecoveryResult, Result};
use crate::integrity::{IntegrityStatus, VerifiedTransaction};
use crate::transaction::{PostCommitFailurePolicy, TransactionReceipt};
const DEFAULT_TIMEOUT: Duration = Duration::from_secs(5);
const DEFAULT_OUTPUT_LIMIT: usize = 64 * 1024;
pub const MAX_STDIN_BYTES: usize = 1024 * 1024;
#[derive(Debug, Clone)]
pub struct CommandSpec {
program: PathBuf,
args: Vec<String>,
current_dir: Option<PathBuf>,
timeout: Duration,
max_output_bytes: usize,
environment: Vec<(String, String)>,
stdin_data: Option<Vec<u8>>,
}
impl CommandSpec {
pub fn new(program: impl Into<PathBuf>) -> Self {
Self {
program: program.into(),
args: Vec::new(),
current_dir: None,
timeout: DEFAULT_TIMEOUT,
max_output_bytes: DEFAULT_OUTPUT_LIMIT,
environment: Vec::new(),
stdin_data: None,
}
}
pub fn arg(mut self, argument: impl Into<String>) -> Self {
self.args.push(argument.into());
self
}
pub fn args<I, S>(mut self, arguments: I) -> Self
where
I: IntoIterator<Item = S>,
S: Into<String>,
{
self.args.extend(arguments.into_iter().map(Into::into));
self
}
pub fn current_dir(mut self, path: impl Into<PathBuf>) -> Self {
self.current_dir = Some(path.into());
self
}
pub fn timeout(mut self, timeout: Duration) -> Self {
self.timeout = timeout;
self
}
pub fn max_output_bytes(mut self, limit: usize) -> Self {
self.max_output_bytes = limit;
self
}
pub fn environment(mut self, name: impl Into<String>, value: impl Into<String>) -> Self {
self.environment.push((name.into(), value.into()));
self
}
pub fn stdin_data(mut self, data: impl Into<Vec<u8>>) -> Self {
self.stdin_data = Some(data.into());
self
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CommandOutput {
exit_code: Option<i32>,
stdout: Vec<u8>,
stderr: Vec<u8>,
timed_out: bool,
output_limited: bool,
}
impl CommandOutput {
pub fn exit_code(&self) -> Option<i32> {
self.exit_code
}
pub fn stdout(&self) -> &[u8] {
&self.stdout
}
pub fn stderr(&self) -> &[u8] {
&self.stderr
}
pub fn timed_out(&self) -> bool {
self.timed_out
}
pub fn output_limited(&self) -> bool {
self.output_limited
}
pub fn success(&self) -> bool {
!self.timed_out && !self.output_limited && self.exit_code == Some(0)
}
}
pub fn run_bounded(spec: &CommandSpec) -> Result<CommandOutput> {
if let Some(data) = &spec.stdin_data {
if data.len() > MAX_STDIN_BYTES {
return Err(Error::InvalidInput(
"stdin data exceeds the bounded stdin bound".into(),
));
}
}
let mut command = Command::new(&spec.program);
command
.args(&spec.args)
.env_clear()
.stdin(if spec.stdin_data.is_some() {
Stdio::piped()
} else {
Stdio::null()
})
.stdout(Stdio::piped())
.stderr(Stdio::piped());
if let Some(current_dir) = &spec.current_dir {
command.current_dir(current_dir);
}
for (name, value) in &spec.environment {
command.env(name, value);
}
let mut child = command
.spawn()
.map_err(|source| Error::io("spawning bounded candidate", source))?;
let deadline = Instant::now() + spec.timeout;
let stdout = child
.stdout
.take()
.ok_or_else(|| Error::CandidateExecution("missing stdout pipe".into()))?;
let stderr = child
.stderr
.take()
.ok_or_else(|| Error::CandidateExecution("missing stderr pipe".into()))?;
let stdin_pipe = if spec.stdin_data.is_some() {
match child.stdin.take() {
Some(pipe) => Some(pipe),
None => {
let _ = child.kill();
let _ = child.wait();
return Err(Error::CandidateExecution("missing stdin pipe".into()));
}
}
} else {
None
};
let (stdout_tx, stdout_rx) = mpsc::channel();
let (stderr_tx, stderr_rx) = mpsc::channel();
let limit = spec.max_output_bytes;
let stdout_thread = thread::spawn(move || {
let _ = stdout_tx.send(read_limited(stdout, limit));
});
let stderr_thread = thread::spawn(move || {
let _ = stderr_tx.send(read_limited(stderr, limit));
});
let (stdin_tx, stdin_rx) = mpsc::channel();
let stdin_thread = stdin_pipe.map(|mut pipe| {
let payload = spec.stdin_data.clone().unwrap_or_default();
thread::spawn(move || {
use std::io::Write as _;
let result = pipe.write_all(&payload).and_then(|()| pipe.flush());
let verdict = WriterVerdict {
observed_at: Instant::now(),
result,
};
let _ = stdin_tx.send(verdict);
})
});
let mut timed_out = false;
let mut output_limited = false;
let mut status = None;
let mut stdout_result = None;
let mut stderr_result = None;
let mut stdout_finished = false;
let mut stderr_finished = false;
let mut stdin_result: Option<WriterVerdict> = None;
let mut stdin_finished = stdin_thread.is_none();
let mut stdin_error: Option<io::Error> = None;
loop {
if !stdout_finished {
stdout_finished = poll_reader(&stdout_rx, &mut stdout_result, &mut output_limited);
}
if !stderr_finished {
stderr_finished = poll_reader(&stderr_rx, &mut stderr_result, &mut output_limited);
}
if !stdin_finished {
stdin_finished = poll_writer(&stdin_rx, &mut stdin_result);
}
if status.is_none() {
status = child
.try_wait()
.map_err(|source| Error::io("polling bounded candidate", source))?;
}
if let Some(verdict) = stdin_result.take() {
match classify_writer_verdict(verdict, deadline) {
WriterVerdictClass::Delivered => {}
WriterVerdictClass::FailedBeforeDeadline(source) => {
stdin_error = Some(source);
if status.is_none() {
let _ = child.kill();
status = child.wait().ok();
}
break;
}
WriterVerdictClass::FailedAtOrAfterDeadline => {
timed_out = true;
if status.is_none() {
let _ = child.kill();
status = child.wait().ok();
}
break;
}
}
}
if output_limited {
if status.is_none() {
let _ = child.kill();
status = child.wait().ok();
}
break;
}
if status.is_some() && stdout_result.is_some() && stderr_result.is_some() && stdin_finished
{
break;
}
if Instant::now() >= deadline {
timed_out = true;
if status.is_none() {
let _ = child.kill();
status = child.wait().ok();
}
break;
}
thread::sleep(Duration::from_millis(2));
}
stdout_finished = poll_reader(&stdout_rx, &mut stdout_result, &mut output_limited);
stderr_finished = poll_reader(&stderr_rx, &mut stderr_result, &mut output_limited);
if !stdin_finished {
stdin_finished = poll_writer(&stdin_rx, &mut stdin_result);
}
if let Some(verdict) = stdin_result.take() {
match classify_writer_verdict(verdict, deadline) {
WriterVerdictClass::Delivered => {}
WriterVerdictClass::FailedBeforeDeadline(source) => {
stdin_error = Some(source);
}
WriterVerdictClass::FailedAtOrAfterDeadline => {}
}
}
if stdout_finished || stdout_thread.is_finished() {
let _ = stdout_thread.join();
}
if stderr_finished || stderr_thread.is_finished() {
let _ = stderr_thread.join();
}
if let Some(writer) = stdin_thread {
if stdin_finished || writer.is_finished() {
let _ = writer.join();
}
}
if let Some(source) = stdin_error {
return Err(Error::io("writing bounded stdin", source));
}
Ok(CommandOutput {
exit_code: status.and_then(|status| status.code()),
stdout: stdout_result
.and_then(|result| result.ok())
.unwrap_or_default(),
stderr: stderr_result
.and_then(|result| result.ok())
.unwrap_or_default(),
timed_out,
output_limited,
})
}
struct WriterVerdict {
observed_at: Instant,
result: io::Result<()>,
}
#[derive(Debug)]
enum WriterVerdictClass {
Delivered,
FailedBeforeDeadline(io::Error),
FailedAtOrAfterDeadline,
}
fn classify_writer_verdict(verdict: WriterVerdict, deadline: Instant) -> WriterVerdictClass {
match verdict.result {
Ok(()) => WriterVerdictClass::Delivered,
Err(source) if verdict.observed_at < deadline => {
WriterVerdictClass::FailedBeforeDeadline(source)
}
Err(_) => WriterVerdictClass::FailedAtOrAfterDeadline,
}
}
fn poll_writer(receiver: &mpsc::Receiver<WriterVerdict>, slot: &mut Option<WriterVerdict>) -> bool {
if slot.is_some() {
return true;
}
match receiver.try_recv() {
Ok(verdict) => {
*slot = Some(verdict);
true
}
Err(mpsc::TryRecvError::Disconnected) => {
*slot = Some(WriterVerdict {
observed_at: Instant::now(),
result: Err(io::Error::other(
"bounded stdin writer ended without a result",
)),
});
true
}
Err(mpsc::TryRecvError::Empty) => false,
}
}
fn poll_reader(
receiver: &mpsc::Receiver<io::Result<Vec<u8>>>,
slot: &mut Option<io::Result<Vec<u8>>>,
output_limited: &mut bool,
) -> bool {
if slot.is_some() {
return true;
}
match receiver.try_recv() {
Ok(result) => {
*output_limited |= result.is_err();
*slot = Some(result);
true
}
Err(mpsc::TryRecvError::Disconnected) => true,
Err(mpsc::TryRecvError::Empty) => false,
}
}
fn read_limited(mut reader: impl Read, limit: usize) -> io::Result<Vec<u8>> {
let mut output = Vec::new();
let mut buffer = [0_u8; 4096];
loop {
let count = reader.read(&mut buffer)?;
if count == 0 {
return Ok(output);
}
if output.len().saturating_add(count) > limit {
return Err(io::Error::other("candidate output limit exceeded"));
}
output.extend_from_slice(&buffer[..count]);
}
}
pub trait CandidateValidator {
fn validate(&self, transaction: &VerifiedTransaction) -> Result<()>;
}
#[derive(Debug, Clone)]
pub struct ExactIdentityValidator {
checks: Vec<IdentityCheck>,
timeout: Duration,
max_output_bytes: usize,
}
#[derive(Debug, Clone)]
struct IdentityCheck {
member: MemberId,
args: Vec<String>,
expected_output: String,
}
impl ExactIdentityValidator {
pub fn new(member: MemberId, expected_output: impl Into<String>) -> Self {
Self {
checks: vec![IdentityCheck {
member,
args: Vec::new(),
expected_output: expected_output.into(),
}],
timeout: DEFAULT_TIMEOUT,
max_output_bytes: DEFAULT_OUTPUT_LIMIT,
}
}
pub fn for_members<I>(members: I, expected_output: impl Into<String>) -> Self
where
I: IntoIterator<Item = MemberId>,
{
let expected_output = expected_output.into();
let checks = members
.into_iter()
.map(|member| IdentityCheck {
member,
args: Vec::new(),
expected_output: expected_output.clone(),
})
.collect();
Self {
checks,
timeout: DEFAULT_TIMEOUT,
max_output_bytes: DEFAULT_OUTPUT_LIMIT,
}
}
pub fn args<I, S>(mut self, args: I) -> Self
where
I: IntoIterator<Item = S>,
S: Into<String>,
{
let args = args.into_iter().map(Into::into).collect::<Vec<_>>();
for check in &mut self.checks {
check.args = args.clone();
}
self
}
pub fn timeout(mut self, timeout: Duration) -> Self {
self.timeout = timeout;
self
}
pub fn max_output_bytes(mut self, limit: usize) -> Self {
self.max_output_bytes = limit;
self
}
}
impl CandidateValidator for ExactIdentityValidator {
fn validate(&self, transaction: &VerifiedTransaction) -> Result<()> {
for check in &self.checks {
let program = transaction.staged_path(&check.member)?;
let output = run_bounded(
&CommandSpec::new(program)
.args(check.args.clone())
.current_dir(transaction.stage_root())
.timeout(self.timeout)
.max_output_bytes(self.max_output_bytes),
)?;
if !output.success()
|| !output.stderr().is_empty()
|| output.stdout() != check.expected_output.as_bytes()
{
return Err(Error::CandidateExecution(format!(
"member {} did not produce the exact expected identity",
check.member
)));
}
}
Ok(())
}
}
#[derive(Debug, Clone)]
pub struct CrossMemberAgreementValidator(ExactIdentityValidator);
impl CrossMemberAgreementValidator {
pub fn new<I>(members: I, expected_output: impl Into<String>) -> Self
where
I: IntoIterator<Item = MemberId>,
{
Self(ExactIdentityValidator::for_members(
members,
expected_output,
))
}
pub fn timeout(mut self, timeout: Duration) -> Self {
self.0 = self.0.timeout(timeout);
self
}
pub fn max_output_bytes(mut self, limit: usize) -> Self {
self.0 = self.0.max_output_bytes(limit);
self
}
}
impl CandidateValidator for CrossMemberAgreementValidator {
fn validate(&self, transaction: &VerifiedTransaction) -> Result<()> {
self.0.validate(transaction)
}
}
pub struct AllValidators<'a> {
validators: Vec<&'a dyn CandidateValidator>,
}
impl<'a> AllValidators<'a> {
pub fn new() -> Self {
Self {
validators: Vec::new(),
}
}
pub fn push(mut self, validator: &'a dyn CandidateValidator) -> Self {
self.validators.push(validator);
self
}
}
impl Default for AllValidators<'_> {
fn default() -> Self {
Self::new()
}
}
impl CandidateValidator for AllValidators<'_> {
fn validate(&self, transaction: &VerifiedTransaction) -> Result<()> {
for validator in &self.validators {
validator.validate(transaction)?;
}
Ok(())
}
}
#[derive(Debug)]
pub struct ValidatedTransaction {
pub(crate) verified: VerifiedTransaction,
}
impl VerifiedTransaction {
pub fn validate(self, validator: &dyn CandidateValidator) -> Result<ValidatedTransaction> {
if self.artifacts().iter().any(|member| {
self.integrity(member.id())
.is_none_or(|result| result.status() != IntegrityStatus::Verified)
}) {
return Err(Error::VerificationFailed(
"candidate execution requires verified integrity evidence for every member".into(),
));
}
validator.validate(&self)?;
Ok(ValidatedTransaction { verified: self })
}
}
impl ValidatedTransaction {
pub fn artifacts(&self) -> &crate::domain::ArtifactSet {
self.verified.artifacts()
}
pub fn staged_path(&self, member: &MemberId) -> Result<std::path::PathBuf> {
self.verified.staged_path(member)
}
pub fn commit(self, ownership: CommitOwnership<'_>) -> Result<TransactionReceipt> {
let digests = self.verified.verified_digests();
self.verified
.prepared
.commit_inner(ownership, &digests, None, None, None)
.map_err(RecoveryError::into_core)
}
pub fn commit_with_post_commit<F, E>(
self,
ownership: CommitOwnership<'_>,
policy: PostCommitFailurePolicy,
check: F,
) -> Result<TransactionReceipt>
where
F: FnOnce() -> std::result::Result<(), E>,
E: std::fmt::Display,
{
let digests = self.verified.verified_digests();
let check = Box::new(move || check().map_err(|error| bounded_display(&error)));
self.verified
.prepared
.commit_inner(ownership, &digests, None, Some((policy, check)), None)
.map_err(RecoveryError::into_core)
}
pub fn commit_with_stale_lock_recovery(
self,
ownership: CommitOwnership<'_>,
recovery: &dyn crate::lock::StaleLockVerifier,
) -> RecoveryResult<TransactionReceipt> {
let digests = self.verified.verified_digests();
self.verified
.prepared
.commit_inner(ownership, &digests, None, None, Some(recovery))
}
pub fn commit_with_post_commit_and_stale_lock_recovery<F, E>(
self,
ownership: CommitOwnership<'_>,
policy: PostCommitFailurePolicy,
recovery: &dyn crate::lock::StaleLockVerifier,
check: F,
) -> RecoveryResult<TransactionReceipt>
where
F: FnOnce() -> std::result::Result<(), E>,
E: std::fmt::Display,
{
let digests = self.verified.verified_digests();
let check = Box::new(move || check().map_err(|error| bounded_display(&error)));
self.verified.prepared.commit_inner(
ownership,
&digests,
None,
Some((policy, check)),
Some(recovery),
)
}
#[cfg(test)]
pub(crate) fn commit_with_fault(
self,
ownership: CommitOwnership<'_>,
fault: crate::transaction::CommitFault,
) -> Result<TransactionReceipt> {
let digests = self.verified.verified_digests();
self.verified
.prepared
.commit_inner(ownership, &digests, Some(fault), None, None)
.map_err(RecoveryError::into_core)
}
#[cfg(test)]
pub(crate) fn commit_with_post_commit_fault(
self,
ownership: CommitOwnership<'_>,
policy: PostCommitFailurePolicy,
fault: crate::transaction::CommitFault,
) -> Result<TransactionReceipt> {
let digests = self.verified.verified_digests();
self.verified
.prepared
.commit_inner(
ownership,
&digests,
Some(fault),
Some((
policy,
Box::new(|| Err("injected post-commit failure".to_string())),
)),
None,
)
.map_err(RecoveryError::into_core)
}
}
fn bounded_display(value: &impl std::fmt::Display) -> String {
struct Bounded(String);
impl std::fmt::Write for Bounded {
fn write_str(&mut self, text: &str) -> std::fmt::Result {
for character in text.chars() {
if self.0.len() + character.len_utf8() > 512 {
break;
}
self.0.push(character);
}
Ok(())
}
}
let mut output = Bounded(String::new());
let _ = std::write!(&mut output, "{value}");
output.0
}
#[cfg(test)]
mod tests {
use super::{classify_writer_verdict, WriterVerdict, WriterVerdictClass};
use std::io;
use std::time::{Duration, Instant};
#[test]
fn candidate_writer_verdict_before_the_deadline_fails_closed() {
let deadline = Instant::now() + Duration::from_secs(1);
let verdict = WriterVerdict {
observed_at: deadline - Duration::from_millis(1),
result: Err(io::Error::other("broken pipe")),
};
match classify_writer_verdict(verdict, deadline) {
WriterVerdictClass::FailedBeforeDeadline(source) => {
assert_eq!(
source.kind(),
io::ErrorKind::Other,
"the underlying failure must be preserved"
);
}
other => panic!("a failure observed before the deadline must fail closed: {other:?}"),
}
}
#[test]
fn candidate_writer_verdict_after_the_deadline_does_not_fail_closed() {
let deadline = Instant::now() + Duration::from_secs(1);
for offset in [Duration::ZERO, Duration::from_millis(1)] {
let verdict = WriterVerdict {
observed_at: deadline + offset,
result: Err(io::Error::other("broken pipe")),
};
assert!(
matches!(
classify_writer_verdict(verdict, deadline),
WriterVerdictClass::FailedAtOrAfterDeadline
),
"a failure observed at or after the deadline must not fail closed"
);
}
}
#[test]
fn candidate_writer_verdict_success_is_a_delivery() {
let deadline = Instant::now() + Duration::from_secs(1);
let verdict = WriterVerdict {
observed_at: deadline + Duration::from_secs(1),
result: Ok(()),
};
assert!(matches!(
classify_writer_verdict(verdict, deadline),
WriterVerdictClass::Delivered
));
}
}