use std::io;
use std::io::IsTerminal;
use std::io::Write;
use std::io::stderr;
use std::mem;
use std::ptr::NonNull;
use std::sync::Arc;
use std::sync::atomic::AtomicBool;
use std::sync::atomic::Ordering;
use hermit::Backend;
use hermit::liteinst_bootstrap::EffectiveFilter;
use tracing::Subscriber;
use tracing::metadata::LevelFilter;
use tracing_subscriber::EnvFilter;
use tracing_subscriber::Layer;
use tracing_subscriber::layer::SubscriberExt;
use tracing_subscriber::util::SubscriberInitExt;
const DEFAULT_TRACE_LEVEL: LevelFilter = LevelFilter::WARN;
pub const LOG_MAX_BYTES_ENV: &str = "HERMIT_LOG_MAX_BYTES";
pub const DEFAULT_LOG_MAX_BYTES: u64 = 1 << 30;
const TRUNCATION_MARKER: &[u8] =
b"\n=== HERMIT LOG TRUNCATED: reached the configured size bound (HERMIT_LOG_MAX_BYTES). \
Output beyond this point was DISCARDED. The run itself continued and was NOT affected. ===\n";
pub fn log_max_bytes() -> Result<u64, String> {
match std::env::var(LOG_MAX_BYTES_ENV) {
Ok(raw) => raw.trim().parse().map_err(|_| {
format!(
"{LOG_MAX_BYTES_ENV}={raw:?} is not a byte count. Set a non-negative integer \
number of bytes, or 0 to disable the bound."
)
}),
Err(_) => Ok(DEFAULT_LOG_MAX_BYTES),
}
}
pub struct BoundedWriter<W: Write> {
inner: W,
remaining: u64,
bounded: bool,
announced: bool,
}
#[derive(Debug)]
struct SharedWriteError {
address: NonNull<AtomicBool>,
}
unsafe impl Send for SharedWriteError {}
unsafe impl Sync for SharedWriteError {}
impl Drop for SharedWriteError {
fn drop(&mut self) {
unsafe {
libc::munmap(self.address.as_ptr().cast(), mem::size_of::<AtomicBool>());
}
}
}
#[derive(Clone, Debug)]
pub struct WriteErrorLatch {
shared: Arc<SharedWriteError>,
}
impl WriteErrorLatch {
pub fn new() -> io::Result<Self> {
let address = unsafe {
libc::mmap(
std::ptr::null_mut(),
mem::size_of::<AtomicBool>(),
libc::PROT_READ | libc::PROT_WRITE,
libc::MAP_SHARED | libc::MAP_ANONYMOUS,
-1,
0,
)
};
if address == libc::MAP_FAILED {
return Err(io::Error::last_os_error());
}
let address = NonNull::new(address.cast::<AtomicBool>())
.expect("mmap returned a non-null non-failure address");
unsafe { address.as_ptr().write(AtomicBool::new(false)) };
Ok(Self {
shared: Arc::new(SharedWriteError { address }),
})
}
fn cell(&self) -> &AtomicBool {
unsafe { self.shared.address.as_ref() }
}
fn record_failure(&self) {
self.cell().store(true, Ordering::Release);
}
pub fn failed(&self) -> bool {
self.cell().load(Ordering::Acquire)
}
}
pub struct LatchedWriter<W> {
inner: W,
latch: WriteErrorLatch,
}
impl<W> LatchedWriter<W> {
pub fn new(inner: W, latch: WriteErrorLatch) -> Self {
Self { inner, latch }
}
}
impl<W: Write> Write for LatchedWriter<W> {
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
match self.inner.write(buf) {
Ok(0) if !buf.is_empty() => {
self.latch.record_failure();
Ok(0)
}
Ok(written) => Ok(written),
Err(error) => {
self.latch.record_failure();
Err(error)
}
}
}
fn flush(&mut self) -> io::Result<()> {
self.inner.flush().inspect_err(|_| {
self.latch.record_failure();
})
}
}
impl<W: Write> BoundedWriter<W> {
pub fn new(inner: W, limit: u64) -> Self {
Self {
inner,
remaining: limit,
bounded: limit != 0,
announced: false,
}
}
}
impl<W: Write> BoundedWriter<W> {
fn announce_truncation(&mut self) -> io::Result<()> {
if !self.announced {
self.announced = true;
self.inner.write_all(TRUNCATION_MARKER)?;
self.inner.flush()?;
}
Ok(())
}
}
impl<W: Write> Write for BoundedWriter<W> {
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
if !self.bounded {
return self.inner.write(buf);
}
if self.remaining == 0 {
self.announce_truncation()?;
return Ok(buf.len());
}
let take = buf.len().min(self.remaining as usize);
let written = self.inner.write(&buf[..take])?;
self.remaining -= written as u64;
if written < take {
return Ok(written);
}
if take < buf.len() {
self.announce_truncation()?;
}
Ok(buf.len())
}
fn flush(&mut self) -> io::Result<()> {
self.inner.flush()
}
}
fn env_filter(level: LevelFilter) -> EnvFilter {
EffectiveFilter::from_default_env(level).into_filter()
}
pub struct TracingGuard {
_worker: Option<tracing_appender::non_blocking::WorkerGuard>,
}
impl TracingGuard {
fn synchronous() -> Self {
Self { _worker: None }
}
}
fn file_subscriber<W: Write + Send + 'static>(
level: LevelFilter,
f: W,
) -> (impl Subscriber, tracing_appender::non_blocking::WorkerGuard) {
let filter = env_filter(level);
let (writer, guard) = tracing_appender::non_blocking(f);
let subscriber = tracing_subscriber::fmt()
.with_env_filter(filter)
.with_writer(writer)
.with_ansi(false)
.finish();
(subscriber, guard)
}
fn sync_file_subscriber<W: Write + Send + 'static>(level: LevelFilter, f: W) -> impl Subscriber {
let filter = env_filter(level);
tracing_subscriber::fmt()
.with_env_filter(filter)
.with_writer(std::sync::Mutex::new(f))
.with_ansi(false)
.finish()
}
pub fn init_sync_file_tracing<W: Write + Send + 'static>(
level: Option<LevelFilter>,
f: W,
backend: Backend,
) -> TracingGuard {
align_tracing_pid_baseline(backend);
let level = level.unwrap_or(DEFAULT_TRACE_LEVEL);
let subscriber = sync_file_subscriber(level, f);
subscriber
.try_init()
.expect("global tracing subscriber to install");
TracingGuard::synchronous()
}
#[must_use = "This function returns a guard that should not be immediately dropped"]
pub fn init_file_tracing<W: Write + Send + 'static>(
level: Option<LevelFilter>,
f: W,
) -> TracingGuard {
let level = level.unwrap_or(DEFAULT_TRACE_LEVEL);
let (subscriber, guard) = file_subscriber(level, f);
subscriber
.try_init()
.expect("global tracing subscriber to install");
TracingGuard {
_worker: Some(guard),
}
}
pub fn init_file_tracing_with_evidence<P, E>(
level: Option<LevelFilter>,
public: P,
evidence: E,
) -> TracingGuard
where
P: Write + Send + 'static,
E: Write + Send + 'static,
{
let (public_writer, guard) = tracing_appender::non_blocking(public);
let public_layer = tracing_subscriber::fmt::layer()
.with_writer(public_writer)
.with_ansi(false)
.with_filter(env_filter(level.unwrap_or(DEFAULT_TRACE_LEVEL)));
let evidence_layer = tracing_subscriber::fmt::layer()
.with_writer(std::sync::Mutex::new(evidence))
.with_ansi(false)
.with_filter(LevelFilter::INFO);
tracing_subscriber::registry()
.with(public_layer)
.with(evidence_layer)
.try_init()
.expect("global tracing subscriber to install");
TracingGuard {
_worker: Some(guard),
}
}
fn align_tracing_pid_baseline(backend: Backend) {
if backend == Backend::Kvm {
return;
}
if unsafe { libc::getpid() } != 1 {
return;
}
let child = unsafe { libc::fork() };
assert!(
child >= 0,
"failed to reserve the tracing PID slot: {}",
io::Error::last_os_error()
);
if child == 0 {
unsafe { libc::_exit(0) };
}
let mut status = 0;
loop {
let waited = unsafe { libc::waitpid(child, &mut status, 0) };
if waited == child {
break;
}
let error = io::Error::last_os_error();
if waited < 0 && error.kind() == io::ErrorKind::Interrupted {
continue;
}
panic!("failed to reap tracing PID-slot process {child}: {error}");
}
assert!(
libc::WIFEXITED(status) && libc::WEXITSTATUS(status) == 0,
"tracing PID-slot process {child} exited unexpectedly: {status:#x}"
);
assert_eq!(
child,
detcore::ROOT_DETPID.as_raw() - 1,
"fresh PID namespace did not reserve the expected tracing slot"
);
}
pub fn init_stderr_tracing_with_evidence<E>(
level: Option<LevelFilter>,
evidence: E,
backend: Backend,
) -> TracingGuard
where
E: Write + Send + 'static,
{
align_tracing_pid_baseline(backend);
let public_layer = tracing_subscriber::fmt::layer()
.with_writer(|| detcore::util::RetryingStderr)
.with_ansi(stderr().is_terminal())
.with_filter(env_filter(level.unwrap_or(DEFAULT_TRACE_LEVEL)));
let evidence_layer = tracing_subscriber::fmt::layer()
.with_writer(std::sync::Mutex::new(evidence))
.with_ansi(false)
.with_filter(LevelFilter::INFO);
tracing_subscriber::registry()
.with(public_layer)
.with(evidence_layer)
.try_init()
.expect("global tracing subscriber to install");
TracingGuard::synchronous()
}
pub fn stderr_subscriber(level: Option<LevelFilter>) -> impl Subscriber {
let level = level.unwrap_or(DEFAULT_TRACE_LEVEL);
let filter = env_filter(level);
tracing_subscriber::fmt()
.with_env_filter(filter)
.with_writer(|| detcore::util::RetryingStderr)
.with_ansi(stderr().is_terminal())
.finish()
}
pub fn init_stderr_tracing(level: Option<LevelFilter>, backend: Backend) {
align_tracing_pid_baseline(backend);
stderr_subscriber(level)
.try_init()
.expect("global tracing subscriber to install")
}
#[cfg(test)]
mod tests {
use std::sync::Mutex;
use reverie::process::Container;
use reverie::process::Mount;
use reverie::process::Namespace;
use super::*;
static ENV_LOCK: Mutex<()> = Mutex::new(());
fn pid_namespace() -> Container {
let mut container = Container::new();
container
.unshare(Namespace::PID)
.map_root()
.mount(Mount::proc().allow_readonly_fallback());
container
}
fn fork_and_observe_child_pid() -> i32 {
let child = unsafe { libc::fork() };
assert!(child >= 0, "fork failed: {}", io::Error::last_os_error());
if child == 0 {
let observed = unsafe { libc::getpid() };
unsafe {
libc::_exit(if observed == detcore::ROOT_DETPID.as_raw() {
0
} else {
1
})
};
}
let mut status = 0;
assert_eq!(unsafe { libc::waitpid(child, &mut status, 0) }, child);
assert!(libc::WIFEXITED(status));
assert_eq!(libc::WEXITSTATUS(status), 0, "child observed the wrong PID");
child
}
fn deny_host_thread_creation() -> io::Result<()> {
let denied = 0x0005_0000 | libc::EPERM as u32; let mut filter = [
libc::sock_filter {
code: 0x20, jt: 0,
jf: 0,
k: 0, },
libc::sock_filter {
code: 0x15, jt: 4,
jf: 0,
k: libc::SYS_clone3 as u32,
},
libc::sock_filter {
code: 0x15,
jt: 0,
jf: 2,
k: libc::SYS_clone as u32,
},
libc::sock_filter {
code: 0x20, jt: 0,
jf: 0,
k: 16, },
libc::sock_filter {
code: 0x45, jt: 1,
jf: 0,
k: libc::CLONE_THREAD as u32,
},
libc::sock_filter {
code: 0x06, jt: 0,
jf: 0,
k: 0x7fff_0000, },
libc::sock_filter {
code: 0x06, jt: 0,
jf: 0,
k: denied,
},
];
let program = libc::sock_fprog {
len: filter.len() as u16,
filter: filter.as_mut_ptr(),
};
unsafe {
if libc::prctl(libc::PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) == -1 {
return Err(io::Error::last_os_error());
}
if libc::prctl(
libc::PR_SET_SECCOMP,
libc::SECCOMP_MODE_FILTER,
&program as *const libc::sock_fprog,
) == -1
{
return Err(io::Error::last_os_error());
}
}
Ok(())
}
#[test]
fn synchronous_pid_alignment_matches_the_file_appender_baseline() {
let synchronous_pid = pid_namespace()
.run(|| {
assert_eq!(unsafe { libc::getpid() }, 1);
align_tracing_pid_baseline(Backend::Ptrace);
fork_and_observe_child_pid()
})
.expect("synchronous baseline namespace failed");
let appender_pid = pid_namespace()
.run(|| {
assert_eq!(unsafe { libc::getpid() }, 1);
let (_subscriber, guard) = file_subscriber(DEFAULT_TRACE_LEVEL, io::sink());
let child = fork_and_observe_child_pid();
drop(guard);
child
})
.expect("file-appender baseline namespace failed");
assert_eq!(synchronous_pid, detcore::ROOT_DETPID.as_raw());
assert_eq!(synchronous_pid, appender_pid);
}
#[test]
fn pid_alignment_succeeds_when_host_thread_creation_is_denied() {
let guest_pid = pid_namespace()
.run(|| {
assert_eq!(unsafe { libc::getpid() }, 1);
deny_host_thread_creation().expect("cannot install host-thread deny filter");
align_tracing_pid_baseline(Backend::Ptrace);
fork_and_observe_child_pid()
})
.expect("thread-free alignment namespace failed");
assert_eq!(guest_pid, detcore::ROOT_DETPID.as_raw());
}
#[test]
fn kvm_pid_alignment_allocates_no_host_task() {
let next_pid = pid_namespace()
.run(|| {
assert_eq!(unsafe { libc::getpid() }, 1);
align_tracing_pid_baseline(Backend::Kvm);
let child = unsafe { libc::fork() };
assert!(child >= 0, "fork failed: {}", io::Error::last_os_error());
if child == 0 {
unsafe { libc::_exit(0) };
}
let mut status = 0;
assert_eq!(unsafe { libc::waitpid(child, &mut status, 0) }, child);
assert!(libc::WIFEXITED(status));
assert_eq!(libc::WEXITSTATUS(status), 0);
child
})
.expect("KVM no-allocation namespace failed");
assert_eq!(next_pid, 2);
}
struct FailingWriter {
fail_write: bool,
}
impl Write for FailingWriter {
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
if self.fail_write {
Err(io::Error::other("injected write failure"))
} else {
Ok(buf.len())
}
}
fn flush(&mut self) -> io::Result<()> {
if self.fail_write {
Ok(())
} else {
Err(io::Error::other("injected flush failure"))
}
}
}
#[test]
fn private_writer_latches_write_and_flush_errors() {
let write_latch = WriteErrorLatch::new().unwrap();
let mut write_fails =
LatchedWriter::new(FailingWriter { fail_write: true }, write_latch.clone());
assert!(write_fails.write_all(b"record").is_err());
assert!(write_latch.failed());
let flush_latch = WriteErrorLatch::new().unwrap();
let mut flush_fails =
LatchedWriter::new(FailingWriter { fail_write: false }, flush_latch.clone());
assert!(flush_fails.flush().is_err());
assert!(flush_latch.failed());
}
#[test]
fn private_writer_latch_is_visible_across_fork() {
let latch = WriteErrorLatch::new().unwrap();
let pid = unsafe { libc::fork() };
assert!(pid >= 0, "fork failed");
if pid == 0 {
latch.record_failure();
unsafe { libc::_exit(0) };
}
let mut status = 0;
assert_eq!(unsafe { libc::waitpid(pid, &mut status, 0) }, pid);
assert_eq!(status, 0);
assert!(
latch.failed(),
"a heap-only latch would lose the child writer failure"
);
}
#[test]
fn bound_truncates_and_announces_exactly_once() {
let mut sink: Vec<u8> = Vec::new();
{
let mut writer = BoundedWriter::new(&mut sink, 100);
for _ in 0..50 {
writer.write_all(&[b'x'; 10]).unwrap();
}
writer.flush().unwrap();
}
let body = sink.iter().filter(|byte| **byte == b'x').count();
let text = String::from_utf8_lossy(&sink);
assert_eq!(body, 100, "wrote past the bound");
assert_eq!(
text.matches("HERMIT LOG TRUNCATED").count(),
1,
"truncation must be announced exactly once, not per write"
);
}
#[test]
fn under_the_bound_nothing_is_truncated_or_announced() {
let mut sink: Vec<u8> = Vec::new();
{
let mut writer = BoundedWriter::new(&mut sink, 1000);
for _ in 0..50 {
writer.write_all(&[b'x'; 10]).unwrap();
}
}
assert_eq!(sink.len(), 500);
assert!(!String::from_utf8_lossy(&sink).contains("HERMIT LOG TRUNCATED"));
}
#[test]
fn a_zero_limit_disables_the_bound() {
let mut sink: Vec<u8> = Vec::new();
{
let mut writer = BoundedWriter::new(&mut sink, 0);
writer.write_all(&[b'z'; 4096]).unwrap();
}
assert_eq!(sink.len(), 4096);
}
#[test]
fn a_single_straddling_write_announces_without_a_following_write() {
let mut sink: Vec<u8> = Vec::new();
{
let mut writer = BoundedWriter::new(&mut sink, 10);
writer.write_all(&[b'q'; 40]).unwrap();
}
let text = String::from_utf8_lossy(&sink);
assert_eq!(
text.matches("HERMIT LOG TRUNCATED").count(),
1,
"a truncated final write must announce itself: {text}"
);
assert_eq!(sink.iter().filter(|b| **b == b'q').count(), 10);
}
#[test]
fn the_written_marker_is_the_text_the_comparator_matches() {
let mut sink: Vec<u8> = Vec::new();
{
let mut writer = BoundedWriter::new(&mut sink, 64);
for _ in 0..20 {
writer
.write_all(b"2022-09-06T14:15:47.000000Z INFO detcore: DETLOG x\n")
.unwrap();
}
writer.flush().unwrap();
}
let written = String::from_utf8(sink).unwrap();
assert!(
detcore::logdiff::log_was_truncated(&written),
"the bounded writer produced {written:?}, which detcore::logdiff does not classify \
as truncated; a truncated pair would silently be compared on its prefix"
);
let mut unbounded: Vec<u8> = Vec::new();
{
let mut writer = BoundedWriter::new(&mut unbounded, 0);
writer
.write_all(b"2022-09-06T14:15:47.000000Z INFO detcore: DETLOG x\n")
.unwrap();
}
assert!(
!detcore::logdiff::log_was_truncated(&String::from_utf8(unbounded).unwrap()),
"an untruncated log must not be classified as truncated"
);
}
#[test]
fn a_malformed_bound_is_an_error_not_a_silent_default() {
let _lock = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner());
let restore = std::env::var(LOG_MAX_BYTES_ENV).ok();
unsafe {
for bad in ["unlimited", "none", "1GiB", "-1", "", "1 000"] {
std::env::set_var(LOG_MAX_BYTES_ENV, bad);
assert!(
log_max_bytes().is_err(),
"{bad:?} must be rejected, not silently read as {DEFAULT_LOG_MAX_BYTES}"
);
}
for (good, expected) in [("0", 0u64), ("4096", 4096), (" 4096 ", 4096)] {
std::env::set_var(LOG_MAX_BYTES_ENV, good);
assert_eq!(
log_max_bytes().unwrap(),
expected,
"{good:?} must be accepted"
);
}
std::env::remove_var(LOG_MAX_BYTES_ENV);
assert_eq!(
log_max_bytes().unwrap(),
DEFAULT_LOG_MAX_BYTES,
"an unset variable is the only thing that means the default"
);
match restore {
Some(v) => std::env::set_var(LOG_MAX_BYTES_ENV, v),
None => std::env::remove_var(LOG_MAX_BYTES_ENV),
}
}
}
#[test]
fn a_write_straddling_the_bound_reports_full_consumption() {
let mut sink: Vec<u8> = Vec::new();
let mut writer = BoundedWriter::new(&mut sink, 5);
assert_eq!(writer.write(&[b'y'; 40]).unwrap(), 40);
}
}