use std::collections::BTreeMap;
use std::fmt::Write as _;
use std::io;
use std::sync::Mutex;
use std::sync::OnceLock;
use std::sync::atomic::AtomicBool;
use std::sync::atomic::AtomicU64;
use std::sync::atomic::Ordering;
use serde::Deserialize;
use serde::Serialize;
use tracing::Event;
use tracing::Level;
use tracing::Metadata;
use tracing::Subscriber;
use tracing::callsite::Callsite;
use tracing::field::Field;
use tracing::field::FieldSet;
use tracing::field::Value;
use tracing::field::Visit;
use tracing::level_filters::LevelFilter;
use tracing::metadata::Kind;
use tracing::span;
use tracing::subscriber::Interest;
#[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize)]
pub struct GuestLogRecord {
pub level: u8,
pub target: String,
pub message: String,
}
pub type CoordinatorCall = fn(
<crate::GlobalState as reverie::GlobalTool>::Request,
) -> io::Result<<crate::GlobalState as reverie::GlobalTool>::Response>;
#[derive(Clone, Copy)]
pub struct OwnChannel {
pub max_level: Level,
pub write: fn(&Event<'_>),
}
const SENT_LEVEL: Level = Level::WARN;
static UNSENT: AtomicU64 = AtomicU64::new(0);
static UNSENT_OWNER: Mutex<Option<crate::types::MmId>> = Mutex::new(None);
static NEXT_SPAN_ID: AtomicU64 = AtomicU64::new(1);
static SENDING: AtomicBool = AtomicBool::new(false);
pub fn level_code(level: &Level) -> u8 {
match *level {
Level::ERROR => 1,
Level::WARN => 2,
Level::INFO => 3,
Level::DEBUG => 4,
Level::TRACE => 5,
}
}
fn level_from_code(code: u8) -> Option<Level> {
match code {
1 => Some(Level::ERROR),
2 => Some(Level::WARN),
3 => Some(Level::INFO),
4 => Some(Level::DEBUG),
5 => Some(Level::TRACE),
_ => None,
}
}
pub fn subscriber(call: CoordinatorCall, own: Option<OwnChannel>) -> impl Subscriber {
GuestLogSubscriber { call, own }
}
pub fn install(call: CoordinatorCall, own: Option<OwnChannel>) -> bool {
tracing::subscriber::set_global_default(subscriber(call, own)).is_ok()
}
pub(crate) fn starts_new_process(mm: crate::types::MmId) {
if !tracing::dispatcher::get_default(|dispatch| dispatch.is::<GuestLogSubscriber>()) {
return;
}
let mut owner = UNSENT_OWNER
.lock()
.unwrap_or_else(|poison| poison.into_inner());
if owner.is_some_and(|owner| owner != mm) {
UNSENT.store(0, Ordering::Release);
SENDING.store(false, Ordering::Release);
}
*owner = Some(mm);
}
pub(crate) fn flush_unsent() {
if UNSENT.load(Ordering::Acquire) == 0 {
return;
}
tracing::dispatcher::get_default(|dispatch| {
if let Some(subscriber) = dispatch.downcast_ref::<GuestLogSubscriber>() {
subscriber.send(None);
}
});
}
pub(crate) async fn flush_unsent_through<R>(rpc: &R)
where
R: reverie::GlobalRPC<crate::GlobalState>,
{
let unsent = UNSENT.swap(0, Ordering::AcqRel);
if unsent == 0 {
return;
}
let answer = rpc
.send_rpc(crate::tool_global::guest_log_request(None, unsent))
.await;
if crate::tool_global::guest_log_reply(answer).is_err() {
UNSENT.fetch_add(unsent, Ordering::AcqRel);
}
}
struct Sending;
impl Drop for Sending {
fn drop(&mut self) {
SENDING.store(false, Ordering::Release);
}
}
struct GuestLogSubscriber {
call: CoordinatorCall,
own: Option<OwnChannel>,
}
impl GuestLogSubscriber {
fn send(&self, record: Option<GuestLogRecord>) {
let counts = u64::from(record.is_some());
if SENDING.swap(true, Ordering::AcqRel) {
UNSENT.fetch_add(counts, Ordering::AcqRel);
return;
}
let _sending = Sending;
let unsent = UNSENT.swap(0, Ordering::AcqRel);
let sent = (self.call)(crate::tool_global::guest_log_request(record, unsent))
.and_then(crate::tool_global::guest_log_reply);
if sent.is_err() {
UNSENT.fetch_add(unsent + counts, Ordering::AcqRel);
}
}
fn own_takes(&self, level: &Level) -> Option<OwnChannel> {
self.own.filter(|own| *level <= own.max_level)
}
fn sends(&self, level: &Level) -> bool {
self.own.is_none() && *level <= SENT_LEVEL
}
}
impl Subscriber for GuestLogSubscriber {
fn max_level_hint(&self) -> Option<LevelFilter> {
Some(match self.own {
Some(own) => own.max_level.into(),
None => SENT_LEVEL.into(),
})
}
fn enabled(&self, metadata: &Metadata<'_>) -> bool {
self.sends(metadata.level()) || self.own_takes(metadata.level()).is_some()
}
fn new_span(&self, _span: &span::Attributes<'_>) -> span::Id {
span::Id::from_u64(NEXT_SPAN_ID.fetch_add(1, Ordering::Relaxed))
}
fn record(&self, _span: &span::Id, _values: &span::Record<'_>) {}
fn record_follows_from(&self, _span: &span::Id, _follows: &span::Id) {}
fn event(&self, event: &Event<'_>) {
let metadata = event.metadata();
if let Some(own) = self.own_takes(metadata.level()) {
(own.write)(event);
return;
}
if !self.sends(metadata.level()) {
return;
}
let mut fields = Fields::default();
event.record(&mut fields);
self.send(Some(GuestLogRecord {
level: level_code(metadata.level()),
target: metadata.target().to_owned(),
message: fields.0,
}));
}
fn enter(&self, _span: &span::Id) {}
fn exit(&self, _span: &span::Id) {}
}
#[derive(Default)]
struct Fields(String);
impl Fields {
fn separate(&mut self) {
if !self.0.is_empty() {
self.0.push(' ');
}
}
}
impl Visit for Fields {
fn record_str(&mut self, field: &Field, value: &str) {
self.separate();
if field.name() == "message" {
self.0.push_str(value);
} else {
let _ = write!(self.0, "{}={value:?}", field.name());
}
}
fn record_debug(&mut self, field: &Field, value: &dyn std::fmt::Debug) {
self.separate();
if field.name() == "message" {
let _ = write!(self.0, "{value:?}");
} else {
let _ = write!(self.0, "{}={value:?}", field.name());
}
}
}
struct GuestCallsite {
metadata: OnceLock<Metadata<'static>>,
}
impl Callsite for GuestCallsite {
fn set_interest(&self, _interest: Interest) {}
fn metadata(&self) -> &Metadata<'_> {
self.metadata
.get()
.expect("a guest callsite's metadata is set before it is registered")
}
}
const MAX_CALLSITES: usize = 1024;
fn callsite(level: Level, target: &str) -> Option<&'static GuestCallsite> {
static CALLSITES: Mutex<BTreeMap<(u8, String), &'static GuestCallsite>> =
Mutex::new(BTreeMap::new());
let mut callsites = CALLSITES
.lock()
.unwrap_or_else(|poison| poison.into_inner());
let key = (level_code(&level), target.to_owned());
if let Some(callsite) = callsites.get(&key) {
return Some(callsite);
}
if callsites.len() >= MAX_CALLSITES {
return None;
}
let callsite: &'static GuestCallsite = Box::leak(Box::new(GuestCallsite {
metadata: OnceLock::new(),
}));
let target: &'static str = Box::leak(target.to_owned().into_boxed_str());
let _ = callsite.metadata.set(Metadata::new(
"guest event",
target,
level,
None,
None,
None,
FieldSet::new(&["message"], tracing::callsite::Identifier(callsite)),
Kind::EVENT,
));
tracing::callsite::register(callsite);
callsites.insert(key, callsite);
Some(callsite)
}
pub(crate) fn emit(record: Option<&GuestLogRecord>, unsent_before: u64) {
if unsent_before > 0 {
tracing::warn!(
"{} Detcore warning(s) from inside the guest could not be sent to the coordinator and are missing",
unsent_before
);
}
let Some(record) = record else {
return;
};
let Some(level) = level_from_code(record.level) else {
return;
};
if level > LevelFilter::current() {
return;
}
let Some(callsite) = callsite(level, &record.target) else {
tracing::warn!(
"(guest record of {} {}, past the callsite limit) {}",
level,
record.target,
record.message
);
return;
};
let metadata = callsite.metadata();
tracing::dispatcher::get_default(|dispatch| {
if !dispatch.enabled(metadata) {
return;
}
let fields = metadata.fields();
let Some(message) = fields.field("message") else {
return;
};
let text = format_args!("{}", record.message);
let values = [(&message, Some(&text as &dyn Value))];
dispatch.event(&Event::new(metadata, &fields.value_set(&values)));
});
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
use super::*;
use crate::tool_global::GlobalRequest;
use crate::tool_global::GlobalResponse;
#[derive(Default)]
struct Capture(Mutex<Vec<(Level, String, String)>>);
impl Subscriber for Capture {
fn enabled(&self, _metadata: &Metadata<'_>) -> bool {
true
}
fn new_span(&self, _span: &span::Attributes<'_>) -> span::Id {
span::Id::from_u64(1)
}
fn record(&self, _span: &span::Id, _values: &span::Record<'_>) {}
fn record_follows_from(&self, _span: &span::Id, _follows: &span::Id) {}
fn event(&self, event: &Event<'_>) {
let mut fields = Fields::default();
event.record(&mut fields);
let metadata = event.metadata();
self.0.lock().unwrap().push((
*metadata.level(),
metadata.target().to_owned(),
fields.0,
));
}
fn enter(&self, _span: &span::Id) {}
fn exit(&self, _span: &span::Id) {}
}
#[test]
fn level_codes_round_trip() {
for level in [
Level::ERROR,
Level::WARN,
Level::INFO,
Level::DEBUG,
Level::TRACE,
] {
assert_eq!(level_from_code(level_code(&level)), Some(level));
}
assert_eq!(level_from_code(0), None);
}
#[test]
fn fields_read_as_the_default_formatter_prints_them() {
let capture = Arc::new(Capture::default());
tracing::subscriber::with_default(capture.clone(), || {
tracing::warn!("cpuid leaf 0x{:x} not in the table", 0x4ffffff0);
tracing::warn!(fd = 3, path = "a b", "cannot restore");
});
let fields: Vec<_> = capture
.0
.lock()
.unwrap()
.iter()
.map(|event| event.2.clone())
.collect();
assert_eq!(
fields,
[
"cpuid leaf 0x4ffffff0 not in the table",
"cannot restore fd=3 path=\"a b\""
]
);
}
#[test]
fn the_coordinator_emits_a_record_at_its_level_and_target_after_its_losses() {
let capture = Arc::new(Capture::default());
tracing::subscriber::with_default(capture.clone(), || {
emit(
Some(&GuestLogRecord {
level: 2,
target: "detcore::syscalls::files".to_owned(),
message: "cannot restore fd=3".to_owned(),
}),
2,
);
emit(None, 1);
emit(None, 0);
});
let events = capture.0.lock().unwrap();
assert_eq!(events.len(), 3, "{events:?}");
let ours = module_path!().trim_end_matches("::tests");
assert_eq!((events[0].0, events[0].1.as_str()), (Level::WARN, ours));
assert!(
events[0]
.2
.starts_with("2 Detcore warning(s) from inside the guest"),
"{events:?}"
);
assert_eq!(
events[1],
(
Level::WARN,
"detcore::syscalls::files".to_owned(),
"cannot restore fd=3".to_owned()
)
);
assert!(
events[2].2.starts_with("1 Detcore warning(s)"),
"{events:?}"
);
}
static SENT: Mutex<Vec<(Option<GuestLogRecord>, u64, bool)>> = Mutex::new(Vec::new());
fn fake_call(
request: <crate::GlobalState as reverie::GlobalTool>::Request,
) -> io::Result<<crate::GlobalState as reverie::GlobalTool>::Response> {
let GlobalRequest::GuestLog(record, unsent) = request.2 else {
panic!("the guest log sends only GuestLog requests: {request:?}");
};
let mut sent = SENT.lock().unwrap();
let refuse = sent.is_empty();
sent.push((record, unsent, refuse));
if refuse {
Err(io::Error::other("refused"))
} else {
Ok((None, GlobalResponse::GuestLog(())))
}
}
static COUNT_TEST_LOCK: Mutex<()> = Mutex::new(());
#[test]
fn the_guest_sends_warnings_counts_what_it_could_not_send_and_flushes_at_exit() {
let _count = COUNT_TEST_LOCK
.lock()
.unwrap_or_else(|poison| poison.into_inner());
use crate::types::DetTid;
use crate::types::MmId;
let root = MmId::initial(DetTid::from_raw(3));
let child = MmId::initial(DetTid::from_raw(7));
UNSENT.store(5, Ordering::Release);
starts_new_process(root);
starts_new_process(child);
assert_eq!(UNSENT.load(Ordering::Acquire), 5);
let subscriber = subscriber(fake_call, None);
tracing::subscriber::with_default(subscriber, || {
*UNSENT_OWNER.lock().unwrap() = None;
starts_new_process(root);
starts_new_process(root);
assert_eq!(UNSENT.load(Ordering::Acquire), 5);
starts_new_process(child);
assert_eq!(UNSENT.load(Ordering::Acquire), 0);
starts_new_process(root);
tracing::warn!("first");
tracing::error!(target: "detcore::fifo_peer", "second");
tracing::info!("not sent");
tracing::debug!("not sent");
tracing::trace!("not sent");
assert!(!tracing::enabled!(Level::INFO));
UNSENT.store(3, Ordering::Release);
flush_unsent();
flush_unsent();
});
let sent = SENT.lock().unwrap();
let summary: Vec<_> = sent
.iter()
.map(|(record, unsent, refused)| {
(
record
.as_ref()
.map(|r| (r.level, r.target.clone(), r.message.clone())),
*unsent,
*refused,
)
})
.collect();
assert_eq!(
summary,
[
(
Some((2, module_path!().to_owned(), "first".to_owned())),
0,
true
),
(
Some((1, "detcore::fifo_peer".to_owned(), "second".to_owned())),
1,
false
),
(None, 3, false),
]
);
assert_eq!(UNSENT.load(Ordering::Acquire), 0);
}
static OWN: Mutex<Vec<(Level, String)>> = Mutex::new(Vec::new());
fn own_write(event: &Event<'_>) {
let mut fields = Fields::default();
event.record(&mut fields);
OWN.lock()
.unwrap()
.push((*event.metadata().level(), fields.0));
}
fn refusing_call(
_request: <crate::GlobalState as reverie::GlobalTool>::Request,
) -> io::Result<<crate::GlobalState as reverie::GlobalTool>::Response> {
panic!("an event the runtime's own channel takes is not sent");
}
#[test]
fn the_own_channel_takes_every_event_up_to_its_level_and_nothing_is_sent() {
let own = OwnChannel {
max_level: Level::INFO,
write: own_write,
};
tracing::subscriber::with_default(subscriber(refusing_call, Some(own)), || {
tracing::error!("an error");
tracing::warn!("a warning");
tracing::info!("DETLOG a record");
tracing::info!("an info");
tracing::debug!("a debug");
tracing::trace!("a trace");
});
assert_eq!(
*OWN.lock().unwrap(),
[
(Level::ERROR, "an error".to_owned()),
(Level::WARN, "a warning".to_owned()),
(Level::INFO, "DETLOG a record".to_owned()),
(Level::INFO, "an info".to_owned()),
]
);
let errors_only = OwnChannel {
max_level: Level::ERROR,
write: own_write,
};
tracing::subscriber::with_default(subscriber(refusing_call, Some(errors_only)), || {
tracing::warn!("above the run's level");
assert!(!tracing::enabled!(Level::WARN));
});
assert_eq!(OWN.lock().unwrap().len(), 4);
}
struct ExitHookRpc {
config: crate::Config,
answers: bool,
sent: Mutex<Vec<GlobalRequest>>,
}
#[reverie::tool]
impl reverie::GlobalRPC<crate::GlobalState> for ExitHookRpc {
async fn send_rpc(
&self,
message: <crate::GlobalState as reverie::GlobalTool>::Request,
) -> <crate::GlobalState as reverie::GlobalTool>::Response {
self.sent.lock().unwrap().push(message.2);
if self.answers {
(None, GlobalResponse::GuestLog(()))
} else {
(None, GlobalResponse::Retired)
}
}
fn config(&self) -> &crate::Config {
&self.config
}
}
#[test]
fn the_exit_hook_sends_a_held_count_through_its_own_connection() {
let _count = COUNT_TEST_LOCK
.lock()
.unwrap_or_else(|poison| poison.into_inner());
let rpc = |answers| ExitHookRpc {
config: Default::default(),
answers,
sent: Mutex::new(Vec::new()),
};
UNSENT.store(0, Ordering::Release);
let idle = rpc(true);
futures::executor::block_on(flush_unsent_through(&idle));
assert!(idle.sent.lock().unwrap().is_empty());
UNSENT.store(4, Ordering::Release);
let refusing = rpc(false);
futures::executor::block_on(flush_unsent_through(&refusing));
assert_eq!(
*refusing.sent.lock().unwrap(),
[GlobalRequest::GuestLog(None, 4)]
);
assert_eq!(UNSENT.load(Ordering::Acquire), 4);
let answering = rpc(true);
futures::executor::block_on(flush_unsent_through(&answering));
assert_eq!(
*answering.sent.lock().unwrap(),
[GlobalRequest::GuestLog(None, 4)]
);
assert_eq!(UNSENT.load(Ordering::Acquire), 0);
}
}