use std::fmt;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::{Arc, Mutex, OnceLock};
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use tokio::sync::{mpsc, oneshot};
use tokio_util::sync::CancellationToken;
use tracing_subscriber::layer::Context as LayerContext;
use tracing_subscriber::Layer;
use crate::audit::{AuditEvent, AuditEventKind, AuditExport, ExportRefused};
use crate::tracing_fields::CappedWriter;
pub const ENDPOINT_VAR: &str = "OTEL_EXPORTER_OTLP_ENDPOINT";
pub const HEADERS_VAR: &str = "OTEL_EXPORTER_OTLP_HEADERS";
pub const PROTOCOL_VAR: &str = "OTEL_EXPORTER_OTLP_PROTOCOL";
pub const SERVICE_NAME_VAR: &str = "OTEL_SERVICE_NAME";
pub const LOGS_ENDPOINT_VAR: &str = "OTEL_EXPORTER_OTLP_LOGS_ENDPOINT";
pub const LOGS_HEADERS_VAR: &str = "OTEL_EXPORTER_OTLP_LOGS_HEADERS";
pub const LOGS_PROTOCOL_VAR: &str = "OTEL_EXPORTER_OTLP_LOGS_PROTOCOL";
pub const ENV_VARS: [&str; 7] = [
ENDPOINT_VAR,
HEADERS_VAR,
PROTOCOL_VAR,
SERVICE_NAME_VAR,
LOGS_ENDPOINT_VAR,
LOGS_HEADERS_VAR,
LOGS_PROTOCOL_VAR,
];
pub const PROTOCOL_HTTP_PROTOBUF: &str = "http/protobuf";
const DEFAULT_SERVICE_NAME: &str = "bugwarden";
const LOGS_PATH: &str = "/v1/logs";
const QUEUE_CAPACITY: usize = 2048;
const MAX_BATCH: usize = 512;
const BATCH_INTERVAL: Duration = Duration::from_millis(500);
const EXPORT_TIMEOUT: Duration = Duration::from_secs(10);
pub const SHUTDOWN_FLUSH_TIMEOUT: Duration = Duration::from_secs(5);
const PROBE_ATTEMPTS: u32 = 5;
const PROBE_BACKOFF: Duration = Duration::from_millis(500);
const SELF_TARGET: &str = "bugwarden::otel";
const NEVER_EXPORTED_TARGETS: [&str; 6] = [
SELF_TARGET,
"reqwest",
"hyper",
"rustls",
"h2",
"tower",
];
fn never_exported(target: &str) -> bool {
NEVER_EXPORTED_TARGETS
.iter()
.any(|prefix| target.starts_with(prefix))
}
const SCOPE_NAME: &str = "bugwarden";
#[derive(Default, Clone)]
pub struct OtelEnv {
pub endpoint: Option<String>,
pub headers: Option<String>,
pub protocol: Option<String>,
pub service_name: Option<String>,
pub logs_endpoint: Option<String>,
pub logs_headers: Option<String>,
pub logs_protocol: Option<String>,
}
impl OtelEnv {
#[must_use]
pub fn from_env() -> Self {
Self {
endpoint: env_value(ENDPOINT_VAR),
headers: env_value(HEADERS_VAR),
protocol: env_value(PROTOCOL_VAR),
service_name: env_value(SERVICE_NAME_VAR),
logs_endpoint: env_value(LOGS_ENDPOINT_VAR),
logs_headers: env_value(LOGS_HEADERS_VAR),
logs_protocol: env_value(LOGS_PROTOCOL_VAR),
}
}
}
fn env_value(var: &str) -> Option<String> {
match std::env::var(var) {
Ok(value) if !value.is_empty() => Some(value),
_ => None,
}
}
#[derive(Clone)]
pub struct ExportConfig {
logs_url: String,
headers: Vec<(String, String)>,
service_name: String,
endpoint_var: &'static str,
headers_var: &'static str,
}
impl ExportConfig {
#[must_use]
pub fn service_name(&self) -> &str {
&self.service_name
}
#[cfg(test)]
pub(crate) fn logs_url(&self) -> &str {
&self.logs_url
}
#[cfg(test)]
pub(crate) fn headers(&self) -> &[(String, String)] {
&self.headers
}
}
pub fn resolve(env: &OtelEnv) -> anyhow::Result<Option<ExportConfig>> {
let (endpoint_var, endpoint, append_path) = match logs_override(env.logs_endpoint.as_deref()) {
Some(endpoint) => (LOGS_ENDPOINT_VAR, endpoint, false),
None => (
ENDPOINT_VAR,
env.endpoint.as_deref().unwrap_or_default().trim(),
true,
),
};
if endpoint.is_empty() {
return Ok(None);
}
let (protocol_var, protocol) = match logs_override(env.logs_protocol.as_deref()) {
Some(protocol) => (LOGS_PROTOCOL_VAR, Some(protocol)),
None => (PROTOCOL_VAR, env.protocol.as_deref().map(str::trim)),
};
match protocol {
None | Some(PROTOCOL_HTTP_PROTOBUF) => {}
Some(_) => anyhow::bail!(
"{protocol_var} selects an OTLP transport this build does not speak; \
only \"{PROTOCOL_HTTP_PROTOBUF}\" is supported"
),
}
if !(endpoint.starts_with("http://") || endpoint.starts_with("https://")) {
anyhow::bail!("{endpoint_var} must be an http:// or https:// URL");
}
let (headers_var, raw_headers) = match logs_override(env.logs_headers.as_deref()) {
Some(raw) => (LOGS_HEADERS_VAR, Some(raw)),
None => (HEADERS_VAR, env.headers.as_deref()),
};
let headers = match raw_headers {
Some(raw) => parse_headers(headers_var, raw)?,
None => Vec::new(),
};
let service_name = match env.service_name.as_deref().map(str::trim) {
Some(name) if !name.is_empty() => name.to_owned(),
_ => DEFAULT_SERVICE_NAME.to_owned(),
};
let logs_url = if append_path {
format!("{}{LOGS_PATH}", endpoint.trim_end_matches('/'))
} else {
endpoint.to_owned()
};
Ok(Some(ExportConfig {
logs_url,
headers,
service_name,
endpoint_var,
headers_var,
}))
}
fn logs_override(value: Option<&str>) -> Option<&str> {
match value.map(str::trim) {
Some(value) if !value.is_empty() => Some(value),
_ => None,
}
}
fn parse_headers(var: &str, raw: &str) -> anyhow::Result<Vec<(String, String)>> {
let mut headers = Vec::new();
for (index, entry) in raw.split(',').enumerate() {
let position = index + 1;
let entry = entry.trim();
if entry.is_empty() {
continue;
}
let Some((key, value)) = entry.split_once('=') else {
anyhow::bail!("{var} entry {position} is not `key=value`");
};
let key = key.trim();
let value = value.trim();
if key.is_empty() || !key.bytes().all(is_header_token) {
anyhow::bail!("{var} entry {position} has no usable header name");
}
if value.is_empty()
|| !value
.bytes()
.all(|b| b == b'\t' || (0x20..=0x7e).contains(&b))
{
anyhow::bail!("{var} entry {position} has no usable header value");
}
headers.push((key.to_owned(), value.to_owned()));
}
Ok(headers)
}
fn is_header_token(b: u8) -> bool {
b.is_ascii_alphanumeric() || b"!#$%&'*+-.^_`|~".contains(&b)
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Severity {
Trace = 1,
Debug = 5,
Info = 9,
Warn = 13,
Error = 17,
}
impl Severity {
fn text(self) -> &'static str {
match self {
Severity::Trace => "TRACE",
Severity::Debug => "DEBUG",
Severity::Info => "INFO",
Severity::Warn => "WARN",
Severity::Error => "ERROR",
}
}
fn of_level(level: &tracing::Level) -> Severity {
match *level {
tracing::Level::TRACE => Severity::Trace,
tracing::Level::DEBUG => Severity::Debug,
tracing::Level::INFO => Severity::Info,
tracing::Level::WARN => Severity::Warn,
tracing::Level::ERROR => Severity::Error,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
enum AttrValue {
Str(String),
Int(i64),
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct LogEntry {
time_unix_nano: u64,
severity: Severity,
body: String,
attrs: Vec<(&'static str, AttrValue)>,
trace: Option<([u8; 16], [u8; 8])>,
}
fn now_unix_nano() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map_or(0, |d| u64::try_from(d.as_nanos()).unwrap_or(u64::MAX))
}
fn audit_entry(event: &AuditEvent, line: &[u8]) -> LogEntry {
let mut attrs: Vec<(&'static str, AttrValue)> = Vec::with_capacity(10);
attrs.push(("bugwarden.stream", AttrValue::Str("audit".to_owned())));
let (kind, severity) = match &event.kind {
AuditEventKind::ToolCall(_) => ("tool_call", Severity::Info),
AuditEventKind::Initialize(_) => ("initialize", Severity::Info),
AuditEventKind::AuditGap(_) => ("audit_gap", Severity::Error),
};
attrs.push(("bugwarden.event", AttrValue::Str(kind.to_owned())));
attrs.push((
"bugwarden.seq",
AttrValue::Int(i64::try_from(event.seq).unwrap_or(i64::MAX)),
));
attrs.push((
"bugwarden.transport",
AttrValue::Str(
match event.session.transport {
crate::audit::TransportKind::Stdio => "stdio",
crate::audit::TransportKind::Http => "http",
}
.to_owned(),
),
));
if let Some(id) = &event.session.id {
attrs.push(("bugwarden.session.id", AttrValue::Str(id.clone())));
}
let mut trace = None;
if let AuditEventKind::ToolCall(call) = &event.kind {
attrs.push(("bugwarden.tool", AttrValue::Str(call.request.tool.clone())));
if let Some(guard) = &call.guard {
attrs.push((
"bugwarden.verdict",
AttrValue::Str(verdict_name(guard.verdict).to_owned()),
));
if let Some(rule) = &guard.rule {
attrs.push(("bugwarden.rule", AttrValue::Str(rule.clone())));
}
}
if let Some(bytes) = call.outcome.response_bytes {
attrs.push((
"bugwarden.response_bytes",
AttrValue::Int(i64::try_from(bytes).unwrap_or(i64::MAX)),
));
}
if let Some(ctx) = &call.trace {
if let (Some(t), Some(s)) =
(hex_bytes::<16>(&ctx.trace_id), hex_bytes::<8>(&ctx.span_id))
{
trace = Some((t, s));
}
}
}
LogEntry {
time_unix_nano: now_unix_nano(),
severity,
body: String::from_utf8_lossy(line).into_owned(),
attrs,
trace,
}
}
fn verdict_name(verdict: crate::audit::Verdict) -> &'static str {
match verdict {
crate::audit::Verdict::Served => "served",
crate::audit::Verdict::ServedFiltered => "served_filtered",
crate::audit::Verdict::Denied => "denied",
crate::audit::Verdict::Refused => "refused",
}
}
fn hex_bytes<const N: usize>(value: &str) -> Option<[u8; N]> {
let bytes = value.as_bytes();
if bytes.len() != N * 2 {
return None;
}
let mut out = [0u8; N];
for (i, slot) in out.iter_mut().enumerate() {
let hi = (bytes[i * 2] as char).to_digit(16)?;
let lo = (bytes[i * 2 + 1] as char).to_digit(16)?;
*slot = u8::try_from(hi * 16 + lo).ok()?;
}
Some(out)
}
mod wire {
pub(super) const LEN: u32 = 2;
pub(super) const I64: u32 = 1;
pub(super) const VARINT: u32 = 0;
pub(super) fn put_varint(buf: &mut Vec<u8>, mut value: u64) {
loop {
let byte = u8::try_from(value & 0x7f).unwrap_or(0);
value >>= 7;
if value == 0 {
buf.push(byte);
return;
}
buf.push(byte | 0x80);
}
}
pub(super) fn put_tag(buf: &mut Vec<u8>, field: u32, wire_type: u32) {
put_varint(buf, u64::from(field << 3 | wire_type));
}
pub(super) fn put_bytes(buf: &mut Vec<u8>, field: u32, value: &[u8]) {
put_tag(buf, field, LEN);
put_varint(buf, value.len() as u64);
buf.extend_from_slice(value);
}
pub(super) fn put_str(buf: &mut Vec<u8>, field: u32, value: &str) {
put_bytes(buf, field, value.as_bytes());
}
pub(super) fn put_varint_field(buf: &mut Vec<u8>, field: u32, value: u64) {
put_tag(buf, field, VARINT);
put_varint(buf, value);
}
pub(super) fn put_fixed64(buf: &mut Vec<u8>, field: u32, value: u64) {
put_tag(buf, field, I64);
buf.extend_from_slice(&value.to_le_bytes());
}
}
fn encode_any_string(value: &str) -> Vec<u8> {
let mut buf = Vec::with_capacity(value.len() + 8);
wire::put_str(&mut buf, 1, value);
buf
}
fn encode_any_int(value: i64) -> Vec<u8> {
let mut buf = Vec::with_capacity(12);
wire::put_varint_field(&mut buf, 3, value as u64);
buf
}
fn encode_kv(key: &str, value: &AttrValue) -> Vec<u8> {
let encoded = match value {
AttrValue::Str(s) => encode_any_string(s),
AttrValue::Int(i) => encode_any_int(*i),
};
let mut buf = Vec::with_capacity(key.len() + encoded.len() + 8);
wire::put_str(&mut buf, 1, key);
wire::put_bytes(&mut buf, 2, &encoded);
buf
}
fn encode_log_record(entry: &LogEntry) -> Vec<u8> {
let mut buf = Vec::with_capacity(entry.body.len() + 128);
wire::put_fixed64(&mut buf, 1, entry.time_unix_nano);
wire::put_varint_field(&mut buf, 2, entry.severity as u64);
wire::put_str(&mut buf, 3, entry.severity.text());
wire::put_bytes(&mut buf, 5, &encode_any_string(&entry.body));
for (key, value) in &entry.attrs {
wire::put_bytes(&mut buf, 6, &encode_kv(key, value));
}
if let Some((trace_id, span_id)) = &entry.trace {
wire::put_bytes(&mut buf, 9, trace_id);
wire::put_bytes(&mut buf, 10, span_id);
}
wire::put_fixed64(&mut buf, 11, entry.time_unix_nano);
buf
}
fn encode_request(service_name: &str, entries: &[LogEntry]) -> Vec<u8> {
let mut scope = Vec::new();
{
let mut inner = Vec::new();
wire::put_str(&mut inner, 1, SCOPE_NAME);
wire::put_str(&mut inner, 2, env!("CARGO_PKG_VERSION"));
wire::put_bytes(&mut scope, 1, &inner);
}
for entry in entries {
wire::put_bytes(&mut scope, 2, &encode_log_record(entry));
}
let mut resource = Vec::new();
wire::put_bytes(
&mut resource,
1,
&encode_kv("service.name", &AttrValue::Str(service_name.to_owned())),
);
let mut resource_logs = Vec::new();
wire::put_bytes(&mut resource_logs, 1, &resource);
wire::put_bytes(&mut resource_logs, 2, &scope);
let mut request = Vec::new();
wire::put_bytes(&mut request, 1, &resource_logs);
request
}
const REASON_QUEUE_FULL: &str = "queue_full";
const REASON_NETWORK: &str = "network";
const REASON_HTTP_STATUS: &str = "http_status";
const REASON_SHUTDOWN: &str = "shutdown";
pub struct Pipeline {
audit_tx: mpsc::Sender<LogEntry>,
diag_tx: mpsc::Sender<LogEntry>,
client: reqwest::Client,
cfg: ExportConfig,
dropped: Arc<AtomicU64>,
log_threshold: Arc<AtomicU64>,
lost: Arc<AtomicU64>,
healthy: Arc<AtomicBool>,
shutdown: CancellationToken,
finished: Mutex<Option<oneshot::Receiver<()>>>,
}
impl fmt::Debug for Pipeline {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("Pipeline")
.field("dropped", &self.dropped.load(Ordering::Relaxed))
.field("healthy", &self.healthy.load(Ordering::Relaxed))
.finish_non_exhaustive()
}
}
impl Pipeline {
pub fn start(cfg: ExportConfig) -> anyhow::Result<Pipeline> {
let client = reqwest::Client::builder()
.timeout(EXPORT_TIMEOUT)
.redirect(reqwest::redirect::Policy::none())
.build()
.map_err(|_| anyhow::anyhow!("the OTLP export HTTP client could not be built"))?;
let (audit_tx, audit_rx) = mpsc::channel(QUEUE_CAPACITY);
let (diag_tx, diag_rx) = mpsc::channel(QUEUE_CAPACITY);
let dropped = Arc::new(AtomicU64::new(0));
let log_threshold = Arc::new(AtomicU64::new(1));
let lost = Arc::new(AtomicU64::new(0));
let healthy = Arc::new(AtomicBool::new(true));
let shutdown = CancellationToken::new();
let (done_tx, done_rx) = oneshot::channel();
let task = ExportTask {
cfg: cfg.clone(),
client: client.clone(),
audit_rx,
diag_rx,
dropped: dropped.clone(),
log_threshold: log_threshold.clone(),
lost: lost.clone(),
healthy: healthy.clone(),
shutdown: shutdown.clone(),
done: done_tx,
};
tokio::spawn(task.run());
Ok(Pipeline {
audit_tx,
diag_tx,
client,
cfg,
dropped,
log_threshold,
lost,
healthy,
shutdown,
finished: Mutex::new(Some(done_rx)),
})
}
pub async fn probe(&self) -> anyhow::Result<()> {
let entry = LogEntry {
time_unix_nano: now_unix_nano(),
severity: Severity::Info,
body: format!(
"bugwarden {} starting: audit records are exported to this collector \
and the server refuses to serve while they cannot be delivered",
env!("CARGO_PKG_VERSION")
),
attrs: vec![
("bugwarden.stream", AttrValue::Str("diagnostics".to_owned())),
("log.target", AttrValue::Str(SELF_TARGET.to_owned())),
],
trace: None,
};
let batch = std::slice::from_ref(&entry);
let mut attempt = 1;
loop {
if post_batch(&self.client, &self.cfg, batch).await.is_ok() {
self.healthy.store(true, Ordering::Relaxed);
return Ok(());
}
if attempt >= PROBE_ATTEMPTS {
self.healthy.store(false, Ordering::Relaxed);
anyhow::bail!(
"the OTLP collector did not accept the startup record after \
{PROBE_ATTEMPTS} attempts; audit records are exported to it and \
this deployment refuses to serve without them — check that the \
endpoint in {} is reachable and that any credential \
in {} is the one it expects",
self.cfg.endpoint_var,
self.cfg.headers_var,
);
}
tokio::time::sleep(PROBE_BACKOFF * attempt).await;
attempt += 1;
}
}
#[must_use]
pub fn dropped(&self) -> u64 {
self.dropped.load(Ordering::Relaxed)
}
#[must_use]
pub fn healthy(&self) -> bool {
self.healthy.load(Ordering::Relaxed)
}
#[must_use]
pub fn audit_exporter(self: &Arc<Self>) -> Arc<dyn AuditExport> {
self.clone()
}
#[must_use]
pub fn diagnostics_layer(slot: Arc<OnceLock<Arc<Pipeline>>>) -> DiagnosticsLayer {
DiagnosticsLayer { slot }
}
pub async fn shutdown(&self) {
self.shutdown.cancel();
let finished = self
.finished
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.take();
if let Some(rx) = finished {
let _ = tokio::time::timeout(SHUTDOWN_FLUSH_TIMEOUT, rx).await;
}
}
fn emit(&self, entry: LogEntry) {
let reason = match self.diag_tx.try_send(entry) {
Ok(()) => return,
Err(mpsc::error::TrySendError::Full(_)) => REASON_QUEUE_FULL,
Err(mpsc::error::TrySendError::Closed(_)) => REASON_SHUTDOWN,
};
note_drops(&self.dropped, &self.log_threshold, 1, reason);
}
}
impl AuditExport for Pipeline {
fn accept(&self, event: &AuditEvent, line: &[u8]) -> Result<(), ExportRefused> {
match self.audit_tx.try_send(audit_entry(event, line)) {
Ok(()) => Ok(()),
Err(_) => {
self.healthy.store(false, Ordering::Relaxed);
Err(ExportRefused)
}
}
}
fn delivery_failing(&self) -> bool {
!self.healthy.load(Ordering::Relaxed)
}
fn take_lost(&self) -> u64 {
self.lost.swap(0, Ordering::Relaxed)
}
}
fn note_drops(dropped: &AtomicU64, threshold: &AtomicU64, count: u64, reason: &'static str) {
if count == 0 {
return;
}
let total = dropped.fetch_add(count, Ordering::Relaxed) + count;
let current = threshold.load(Ordering::Relaxed);
if total < current {
return;
}
let next = if total.is_power_of_two() {
total.saturating_mul(2)
} else {
total.checked_next_power_of_two().unwrap_or(u64::MAX)
};
if threshold
.compare_exchange(current, next, Ordering::Relaxed, Ordering::Relaxed)
.is_ok()
{
tracing::warn!(
dropped = total,
reason,
"otlp export is dropping diagnostic records"
);
}
}
async fn post_batch(
client: &reqwest::Client,
cfg: &ExportConfig,
batch: &[LogEntry],
) -> Result<(), &'static str> {
let body = encode_request(&cfg.service_name, batch);
let mut request = client
.post(&cfg.logs_url)
.header("content-type", "application/x-protobuf")
.body(body);
for (name, value) in &cfg.headers {
request = request.header(name.as_str(), value.as_str());
}
match request.send().await {
Ok(response) if response.status().is_success() => Ok(()),
Ok(_) => Err(REASON_HTTP_STATUS),
Err(_) => Err(REASON_NETWORK),
}
}
fn note_delivery(healthy: &AtomicBool, ok: bool) {
let was = healthy.swap(ok, Ordering::Relaxed);
if was && !ok {
tracing::warn!(
"otlp delivery is failing; the audit fail mode now gates tool calls \
until a delivery succeeds"
);
} else if !was && ok {
tracing::info!("otlp delivery recovered");
}
}
struct ExportTask {
cfg: ExportConfig,
client: reqwest::Client,
audit_rx: mpsc::Receiver<LogEntry>,
diag_rx: mpsc::Receiver<LogEntry>,
dropped: Arc<AtomicU64>,
log_threshold: Arc<AtomicU64>,
lost: Arc<AtomicU64>,
healthy: Arc<AtomicBool>,
shutdown: CancellationToken,
done: oneshot::Sender<()>,
}
impl ExportTask {
async fn run(mut self) {
let mut audit_batch: Vec<LogEntry> = Vec::with_capacity(MAX_BATCH);
let mut diag_batch: Vec<LogEntry> = Vec::with_capacity(MAX_BATCH);
loop {
let deadline = tokio::time::Instant::now() + BATCH_INTERVAL;
let mut stopping = false;
loop {
tokio::select! {
biased;
() = self.shutdown.cancelled() => { stopping = true; break; }
entry = self.audit_rx.recv() => match entry {
Some(entry) => {
audit_batch.push(entry);
if audit_batch.len() >= MAX_BATCH {
break;
}
}
None => { stopping = true; break; }
},
entry = self.diag_rx.recv() => match entry {
Some(entry) => {
diag_batch.push(entry);
if diag_batch.len() >= MAX_BATCH {
break;
}
}
None => { stopping = true; break; }
},
() = tokio::time::sleep_until(deadline) => break,
}
}
self.flush_audit(&mut audit_batch).await;
self.flush_diag(&mut diag_batch).await;
if stopping {
self.audit_rx.close();
self.diag_rx.close();
while let Ok(entry) = self.audit_rx.try_recv() {
audit_batch.push(entry);
if audit_batch.len() >= MAX_BATCH {
self.flush_audit(&mut audit_batch).await;
}
}
while let Ok(entry) = self.diag_rx.try_recv() {
diag_batch.push(entry);
if diag_batch.len() >= MAX_BATCH {
self.flush_diag(&mut diag_batch).await;
}
}
self.flush_audit(&mut audit_batch).await;
self.flush_diag(&mut diag_batch).await;
break;
}
}
let _ = self.done.send(());
}
async fn flush_audit(&self, batch: &mut Vec<LogEntry>) {
if batch.is_empty() {
return;
}
match post_batch(&self.client, &self.cfg, batch).await {
Ok(()) => note_delivery(&self.healthy, true),
Err(_) => {
self.lost.fetch_add(batch.len() as u64, Ordering::Relaxed);
note_delivery(&self.healthy, false);
}
}
batch.clear();
}
async fn flush_diag(&self, batch: &mut Vec<LogEntry>) {
if batch.is_empty() {
return;
}
match post_batch(&self.client, &self.cfg, batch).await {
Ok(()) => {}
Err(reason) => {
note_drops(
&self.dropped,
&self.log_threshold,
batch.len() as u64,
reason,
);
note_delivery(&self.healthy, false);
}
}
batch.clear();
}
}
pub struct DiagnosticsLayer {
slot: Arc<OnceLock<Arc<Pipeline>>>,
}
struct BodyVisitor {
message: String,
fields: String,
log_target: Option<String>,
}
fn is_log_bridge_field(name: &str) -> bool {
matches!(
name,
"log.target" | "log.module_path" | "log.file" | "log.line"
)
}
impl BodyVisitor {
fn open_field(&mut self, name: &str) -> CappedWriter<'_, String> {
if !self.fields.is_empty() {
self.fields.push(' ');
}
self.fields.push_str(name);
self.fields.push('=');
CappedWriter::new(&mut self.fields)
}
}
impl tracing::field::Visit for BodyVisitor {
fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn fmt::Debug) {
use std::fmt::Write as _;
if field.name() == "log.target" {
self.log_target = Some(format!("{value:?}").trim_matches('"').to_owned());
return;
}
if is_log_bridge_field(field.name()) {
return;
}
if field.name() == "message" {
let _ = write!(CappedWriter::new(&mut self.message), "{value:?}");
} else {
let _ = write!(self.open_field(field.name()), "{value:?}");
}
}
fn record_str(&mut self, field: &tracing::field::Field, value: &str) {
use std::fmt::Write as _;
if field.name() == "log.target" {
self.log_target = Some(value.to_owned());
return;
}
if is_log_bridge_field(field.name()) {
return;
}
if field.name() == "message" {
let _ = CappedWriter::new(&mut self.message).write_str(value);
} else {
let _ = self.open_field(field.name()).write_str(value);
}
}
}
impl<S: tracing::Subscriber> Layer<S> for DiagnosticsLayer {
fn on_event(&self, event: &tracing::Event<'_>, _ctx: LayerContext<'_, S>) {
let meta = event.metadata();
if never_exported(meta.target()) {
return;
}
let Some(pipeline) = self.slot.get() else {
return;
};
let mut visitor = BodyVisitor {
message: String::new(),
fields: String::new(),
log_target: None,
};
event.record(&mut visitor);
let target = visitor
.log_target
.as_deref()
.unwrap_or_else(|| meta.target());
if never_exported(target) {
return;
}
let body = if visitor.fields.is_empty() {
visitor.message
} else if visitor.message.is_empty() {
visitor.fields
} else {
format!("{} {}", visitor.message, visitor.fields)
};
pipeline.emit(LogEntry {
time_unix_nano: now_unix_nano(),
severity: Severity::of_level(meta.level()),
body,
attrs: vec![
("bugwarden.stream", AttrValue::Str("diagnostics".to_owned())),
("log.target", AttrValue::Str(target.to_owned())),
],
trace: None,
});
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::audit::{
AuditEvent, AuditEventKind, ClientInfo, GuardInfo, InitializeEvent, OutcomeClass,
OutcomeInfo, RequestInfo, SessionInfo, ToolCallEvent, TraceContext, TransportKind, Verdict,
};
use crate::refused::refused_base_url;
use crate::testlog::{assert_logged, capture_logs};
fn env(endpoint: &str) -> OtelEnv {
OtelEnv {
endpoint: Some(endpoint.to_owned()),
..OtelEnv::default()
}
}
const NO_COLLECTOR: &str = "http://127.0.0.1:1";
fn dead_collector() -> (String, String, String) {
let base = refused_base_url();
let addr: std::net::SocketAddr = base
.trim_start_matches("http://")
.parse()
.expect("the probed base is `http://` and a socket address");
let (host, port) = (addr.ip().to_string(), format!(":{}", addr.port()));
(base, host, port)
}
#[test]
fn unset_endpoint_turns_export_off() {
let resolved = resolve(&OtelEnv::default()).expect("an empty environment resolves");
assert!(
resolved.is_none(),
"no endpoint must mean no export configuration at all"
);
}
#[test]
fn empty_endpoint_turns_export_off() {
let resolved = resolve(&OtelEnv {
endpoint: Some(String::new()),
..OtelEnv::default()
})
.expect("an emptied endpoint resolves");
assert!(resolved.is_none(), "an emptied endpoint must read as unset");
let resolved = resolve(&OtelEnv {
endpoint: Some(" ".to_owned()),
..OtelEnv::default()
})
.expect("a blank endpoint resolves");
assert!(resolved.is_none(), "a blank endpoint must read as unset");
}
#[test]
fn an_endpoint_gains_the_logs_path_and_the_default_service_name() {
let cfg = resolve(&env("http://collector.example:4318"))
.expect("resolves")
.expect("export is on");
assert_eq!(cfg.logs_url(), "http://collector.example:4318/v1/logs");
assert_eq!(cfg.service_name(), "bugwarden");
let cfg = resolve(&env("http://collector.example:4318/"))
.expect("resolves")
.expect("export is on");
assert_eq!(cfg.logs_url(), "http://collector.example:4318/v1/logs");
}
#[test]
fn service_name_is_taken_from_the_environment_when_set() {
let cfg = resolve(&OtelEnv {
service_name: Some(" bugwarden-edge ".to_owned()),
..env("http://c:4318")
})
.expect("resolves")
.expect("export is on");
assert_eq!(cfg.service_name(), "bugwarden-edge");
}
#[test]
fn the_only_accepted_protocol_is_http_protobuf() {
for accepted in [None, Some("http/protobuf".to_owned())] {
let cfg = resolve(&OtelEnv {
protocol: accepted.clone(),
..env("http://c:4318")
})
.expect("http/protobuf and an unset protocol resolve");
assert!(cfg.is_some(), "{accepted:?} must leave export on");
}
for rejected in ["grpc", "http/json", "HTTP/PROTOBUF", ""] {
let err = resolve(&OtelEnv {
protocol: Some(rejected.to_owned()),
..env("http://c:4318")
})
.err()
.expect("only http/protobuf is spoken");
let text = format!("{err}");
assert!(
text.contains(PROTOCOL_VAR) && text.contains(PROTOCOL_HTTP_PROTOBUF),
"the error must name the variable and the accepted value: {text}"
);
}
}
#[test]
fn a_bad_protocol_without_an_endpoint_is_not_an_error() {
let resolved = resolve(&OtelEnv {
protocol: Some("grpc".to_owned()),
..OtelEnv::default()
})
.expect("no endpoint decides everything");
assert!(resolved.is_none());
}
#[test]
fn an_endpoint_must_be_an_http_url() {
let err = resolve(&env("collector.example:4318"))
.err()
.expect("a bare authority is not a URL");
assert!(format!("{err}").contains(ENDPOINT_VAR));
assert!(resolve(&env("https://c:4318"))
.expect("https resolves")
.is_some());
}
#[test]
fn headers_parse_into_pairs_and_are_taken_verbatim() {
let cfg = resolve(&OtelEnv {
headers: Some(" authorization=Bearer abc%20def , x-tenant=acme ".to_owned()),
..env("http://c:4318")
})
.expect("resolves")
.expect("export is on");
assert_eq!(
cfg.headers(),
[
("authorization".to_owned(), "Bearer abc%20def".to_owned()),
("x-tenant".to_owned(), "acme".to_owned()),
],
"values are used as given; percent-encoding is not decoded"
);
let cfg = resolve(&OtelEnv {
headers: Some("a=1,,b=2,".to_owned()),
..env("http://c:4318")
})
.expect("resolves")
.expect("export is on");
assert_eq!(
cfg.headers(),
[
("a".to_owned(), "1".to_owned()),
("b".to_owned(), "2".to_owned()),
]
);
}
#[test]
fn the_logs_specific_variables_override_the_general_ones() {
let cfg = resolve(&OtelEnv {
logs_endpoint: Some("http://logs.example:4318/otlp/v1/logs".to_owned()),
logs_headers: Some("x-logs=1".to_owned()),
headers: Some("x-general=1".to_owned()),
..env("http://general.example:4318")
})
.expect("resolves")
.expect("export is on");
assert_eq!(cfg.logs_url(), "http://logs.example:4318/otlp/v1/logs");
assert_eq!(cfg.headers(), [("x-logs".to_owned(), "1".to_owned())]);
let cfg = resolve(&OtelEnv {
logs_endpoint: Some("http://logs.example:4318/v1/logs".to_owned()),
..OtelEnv::default()
})
.expect("resolves")
.expect("a logs endpoint alone means export is on");
assert_eq!(cfg.logs_url(), "http://logs.example:4318/v1/logs");
let cfg = resolve(&OtelEnv {
logs_endpoint: Some(String::new()),
..env("http://general.example:4318")
})
.expect("resolves")
.expect("export is on");
assert_eq!(cfg.logs_url(), "http://general.example:4318/v1/logs");
let err = resolve(&OtelEnv {
logs_protocol: Some("grpc".to_owned()),
protocol: Some(PROTOCOL_HTTP_PROTOBUF.to_owned()),
..env("http://c:4318")
})
.err()
.expect("grpc must be refused wherever it came from");
assert!(
format!("{err}").contains(LOGS_PROTOCOL_VAR),
"the error must name the logs-specific variable that won, not its general twin: {err}"
);
}
#[test]
fn a_malformed_header_entry_is_refused_without_echoing_it() {
let secret = "Bearer super-secret-token";
let err = resolve(&OtelEnv {
headers: Some(format!("x-a=1,{secret}")),
..env("http://c:4318")
})
.err()
.expect("an entry without `=` is refused");
let text = format!("{err}");
assert!(
text.contains(HEADERS_VAR) && text.contains('2'),
"the error must name the variable and the position: {text}"
);
assert!(
!text.contains("super-secret-token"),
"the error must never echo header material: {text}"
);
}
#[test]
fn an_unusable_header_name_or_value_is_refused() {
for bad in ["bad name=1", "x=", "=value", "no-equals-sign", "x=va\nlue"] {
let err = resolve(&OtelEnv {
headers: Some(bad.to_owned()),
..env("http://c:4318")
})
.err()
.unwrap_or_else(|| panic!("{bad:?} must be refused"));
assert!(format!("{err}").contains(HEADERS_VAR));
}
}
fn session() -> SessionInfo {
SessionInfo {
id: Some("sess-1".to_owned()),
transport: TransportKind::Http,
remote: Some("192.0.2.7:52611".to_owned()),
}
}
fn tool_call(trace: Option<TraceContext>) -> AuditEventKind {
AuditEventKind::ToolCall(Box::new(ToolCallEvent {
client: ClientInfo {
name: Some("agent".to_owned()),
version: None,
principal: None,
work_context: None,
},
trace,
request: RequestInfo {
tool: "bug_info".to_owned(),
id: Some("3".to_owned()),
params: std::collections::BTreeMap::new(),
},
guard: Some(GuardInfo {
verdict: Verdict::Denied,
rule: Some("embargo".to_owned()),
policy_hash: None,
suppressed_ids_count: 1,
suppressed_other_count: 0,
suppressed_ids: vec![7],
redacted_fields: Vec::new(),
scan: None,
}),
upstream: None,
outcome: OutcomeInfo {
class: OutcomeClass::Ok,
duration_ms: 3,
response_bytes: Some(87),
},
}))
}
fn event(kind: AuditEventKind) -> AuditEvent {
AuditEvent {
v: crate::audit::SCHEMA_VERSION,
ts: "2026-08-18T00:00:00.000Z".to_owned(),
seq: 42,
session: session(),
kind,
}
}
fn attr<'a>(entry: &'a LogEntry, key: &str) -> Option<&'a AttrValue> {
entry.attrs.iter().find(|(k, _)| *k == key).map(|(_, v)| v)
}
#[test]
fn an_audit_entry_carries_the_line_verbatim_as_its_body() {
let line = br#"{"v":1,"seq":42}"#;
let entry = audit_entry(&event(tool_call(None)), line);
assert_eq!(
entry.body.as_bytes(),
line,
"the body must be the file's bytes, unchanged"
);
}
#[test]
fn an_audit_entry_carries_the_documented_attributes() {
let entry = audit_entry(&event(tool_call(None)), b"{}");
assert_eq!(
attr(&entry, "bugwarden.stream"),
Some(&AttrValue::Str("audit".to_owned()))
);
assert_eq!(
attr(&entry, "bugwarden.event"),
Some(&AttrValue::Str("tool_call".to_owned()))
);
assert_eq!(attr(&entry, "bugwarden.seq"), Some(&AttrValue::Int(42)));
assert_eq!(
attr(&entry, "bugwarden.transport"),
Some(&AttrValue::Str("http".to_owned()))
);
assert_eq!(
attr(&entry, "bugwarden.session.id"),
Some(&AttrValue::Str("sess-1".to_owned()))
);
assert_eq!(
attr(&entry, "bugwarden.tool"),
Some(&AttrValue::Str("bug_info".to_owned()))
);
assert_eq!(
attr(&entry, "bugwarden.verdict"),
Some(&AttrValue::Str("denied".to_owned()))
);
assert_eq!(
attr(&entry, "bugwarden.rule"),
Some(&AttrValue::Str("embargo".to_owned()))
);
assert_eq!(
attr(&entry, "bugwarden.response_bytes"),
Some(&AttrValue::Int(87))
);
}
#[test]
fn an_unmeasured_response_exports_no_size_attribute() {
let mut kind = tool_call(None);
let AuditEventKind::ToolCall(call) = &mut kind else {
unreachable!()
};
call.outcome.response_bytes = None;
let entry = audit_entry(&event(kind), b"{}");
assert_eq!(attr(&entry, "bugwarden.response_bytes"), None);
}
#[test]
fn severity_follows_the_event_kind() {
assert_eq!(
audit_entry(&event(tool_call(None)), b"{}").severity,
Severity::Info
);
assert_eq!(
audit_entry(
&event(AuditEventKind::Initialize(InitializeEvent {
client: ClientInfo {
name: None,
version: None,
principal: None,
work_context: None
},
protocol_version: None,
})),
b"{}"
)
.severity,
Severity::Info
);
assert_eq!(
audit_entry(
&event(AuditEventKind::AuditGap(crate::audit::AuditGapEvent {
dropped: 2,
reason: crate::audit::GapReason::WriteError,
})),
b"{}"
)
.severity,
Severity::Error,
"a gap in the record stream is an error, not an info line"
);
}
#[test]
fn trace_ids_reach_the_record_as_raw_bytes() {
let entry = audit_entry(
&event(tool_call(Some(TraceContext {
trace_id: "4bf92f3577b34da6a3ce929d0e0e4736".to_owned(),
span_id: "00f067aa0ba902b7".to_owned(),
}))),
b"{}",
);
let (trace_id, span_id) = entry.trace.expect("a valid traceparent must be exported");
assert_eq!(
trace_id,
[
0x4b, 0xf9, 0x2f, 0x35, 0x77, 0xb3, 0x4d, 0xa6, 0xa3, 0xce, 0x92, 0x9d, 0x0e, 0x0e,
0x47, 0x36
]
);
assert_eq!(span_id, [0x00, 0xf0, 0x67, 0xaa, 0x0b, 0xa9, 0x02, 0xb7]);
}
#[test]
fn a_record_without_trace_context_exports_no_ids() {
assert!(audit_entry(&event(tool_call(None)), b"{}").trace.is_none());
}
#[test]
fn hex_of_the_wrong_width_or_alphabet_decodes_to_nothing() {
assert!(hex_bytes::<8>("00f067aa0ba902b").is_none());
assert!(hex_bytes::<8>("00f067aa0ba902b77").is_none());
assert!(hex_bytes::<8>("00f067aa0ba902bz").is_none());
assert!(hex_bytes::<8>("").is_none());
}
fn fields(buf: &[u8]) -> Vec<(u32, u32, Vec<u8>)> {
let mut out = Vec::new();
let mut i = 0usize;
while i < buf.len() {
let (tag, used) = read_varint(&buf[i..]).expect("a tag");
i += used;
let field = u32::try_from(tag >> 3).expect("field number");
let wire_type = u32::try_from(tag & 7).expect("wire type");
match wire_type {
0 => {
let (value, used) = read_varint(&buf[i..]).expect("a varint");
out.push((field, wire_type, value.to_le_bytes().to_vec()));
i += used;
}
1 => {
out.push((field, wire_type, buf[i..i + 8].to_vec()));
i += 8;
}
2 => {
let (len, used) = read_varint(&buf[i..]).expect("a length");
i += used;
let len = usize::try_from(len).expect("a sane length");
out.push((field, wire_type, buf[i..i + len].to_vec()));
i += len;
}
5 => {
out.push((field, wire_type, buf[i..i + 4].to_vec()));
i += 4;
}
other => panic!("unexpected wire type {other}"),
}
}
out
}
fn read_varint(buf: &[u8]) -> Option<(u64, usize)> {
let mut value = 0u64;
for (i, byte) in buf.iter().enumerate().take(10) {
value |= u64::from(byte & 0x7f) << (7 * i);
if byte & 0x80 == 0 {
return Some((value, i + 1));
}
}
None
}
fn only(buf: &[u8], field: u32) -> Vec<u8> {
fields(buf)
.into_iter()
.find(|(f, _, _)| *f == field)
.unwrap_or_else(|| panic!("field {field} must be present"))
.2
}
fn all(buf: &[u8], field: u32) -> Vec<Vec<u8>> {
fields(buf)
.into_iter()
.filter(|(f, _, _)| *f == field)
.map(|(_, _, v)| v)
.collect()
}
#[test]
fn varints_encode_the_way_protobuf_reads_them() {
for value in [0u64, 1, 127, 128, 300, 16_383, 16_384, u64::MAX] {
let mut buf = Vec::new();
wire::put_varint(&mut buf, value);
assert_eq!(read_varint(&buf), Some((value, buf.len())), "{value}");
}
let mut buf = Vec::new();
wire::put_varint(&mut buf, 300);
assert_eq!(buf, [0xac, 0x02]);
}
#[test]
fn a_request_nests_resource_scope_and_records_as_otlp_expects() {
let entry = audit_entry(&event(tool_call(None)), b"{\"seq\":42}");
let request = encode_request("bugwarden", std::slice::from_ref(&entry));
let resource_logs = only(&request, 1);
let resource = only(&resource_logs, 1);
let service_kv = only(&resource, 1);
assert_eq!(only(&service_kv, 1), b"service.name");
assert_eq!(only(&only(&service_kv, 2), 1), b"bugwarden");
let scope_logs = only(&resource_logs, 2);
let scope = only(&scope_logs, 1);
assert_eq!(only(&scope, 1), SCOPE_NAME.as_bytes());
assert_eq!(only(&scope, 2), env!("CARGO_PKG_VERSION").as_bytes());
let records = all(&scope_logs, 2);
assert_eq!(records.len(), 1);
let record = &records[0];
assert_eq!(only(record, 2)[0], Severity::Info as u8);
assert_eq!(only(record, 3), b"INFO");
assert_eq!(only(&only(record, 5), 1), b"{\"seq\":42}");
assert!(
all(record, 6).len() >= 7,
"every documented attribute must be encoded"
);
let stamp = entry.time_unix_nano.to_le_bytes().to_vec();
assert_eq!(only(record, 1), stamp, "time_unix_nano is field 1");
assert_eq!(
only(record, 11),
stamp,
"observed_time_unix_nano is field 11"
);
assert!(
!fields(record).iter().any(|(f, _, _)| *f == 4),
"field 4 is reserved in the OTLP LogRecord and must carry nothing"
);
}
#[test]
fn every_severity_keeps_its_otlp_number_and_text() {
for (severity, number, text) in [
(Severity::Trace, 1u8, "TRACE"),
(Severity::Debug, 5, "DEBUG"),
(Severity::Info, 9, "INFO"),
(Severity::Warn, 13, "WARN"),
(Severity::Error, 17, "ERROR"),
] {
assert_eq!(severity as u8, number, "{text} is OTLP severity {number}");
assert_eq!(severity.text(), text);
}
let gap = audit_entry(
&event(AuditEventKind::AuditGap(crate::audit::AuditGapEvent {
dropped: 2,
reason: crate::audit::GapReason::WriteError,
})),
b"{}",
);
let record = all(&only(&only(&request_of(&gap), 1), 2), 2).remove(0);
assert_eq!(only(&record, 2)[0], 17, "an audit_gap is ERROR on the wire");
assert_eq!(only(&record, 3), b"ERROR");
}
#[test]
fn a_batch_becomes_one_request_with_one_record_each() {
let entries: Vec<LogEntry> = (0..3)
.map(|_| audit_entry(&event(tool_call(None)), b"{}"))
.collect();
let request = encode_request("bugwarden", &entries);
let scope_logs = only(&only(&request, 1), 2);
assert_eq!(all(&scope_logs, 2).len(), 3);
}
#[test]
fn trace_ids_are_encoded_as_the_fixed_width_byte_fields() {
let entry = audit_entry(
&event(tool_call(Some(TraceContext {
trace_id: "4bf92f3577b34da6a3ce929d0e0e4736".to_owned(),
span_id: "00f067aa0ba902b7".to_owned(),
}))),
b"{}",
);
let record = all(&only(&only(&request_of(&entry), 1), 2), 2).remove(0);
assert_eq!(only(&record, 9).len(), 16);
assert_eq!(only(&record, 10).len(), 8);
assert_eq!(only(&record, 9)[0], 0x4b);
assert_eq!(only(&record, 10)[1], 0xf0);
}
fn request_of(entry: &LogEntry) -> Vec<u8> {
encode_request("bugwarden", std::slice::from_ref(entry))
}
mod otlp {
#[derive(Clone, PartialEq, ::prost::Message)]
pub(super) struct ExportLogsServiceRequest {
#[prost(message, repeated, tag = "1")]
pub(super) resource_logs: Vec<ResourceLogs>,
}
#[derive(Clone, PartialEq, ::prost::Message)]
pub(super) struct ResourceLogs {
#[prost(message, optional, tag = "1")]
pub(super) resource: Option<Resource>,
#[prost(message, repeated, tag = "2")]
pub(super) scope_logs: Vec<ScopeLogs>,
#[prost(string, tag = "3")]
pub(super) schema_url: String,
}
#[derive(Clone, PartialEq, ::prost::Message)]
pub(super) struct ScopeLogs {
#[prost(message, optional, tag = "1")]
pub(super) scope: Option<InstrumentationScope>,
#[prost(message, repeated, tag = "2")]
pub(super) log_records: Vec<LogRecord>,
#[prost(string, tag = "3")]
pub(super) schema_url: String,
}
#[derive(Clone, PartialEq, ::prost::Message)]
pub(super) struct LogRecord {
#[prost(fixed64, tag = "1")]
pub(super) time_unix_nano: u64,
#[prost(int32, tag = "2")]
pub(super) severity_number: i32,
#[prost(string, tag = "3")]
pub(super) severity_text: String,
#[prost(message, optional, tag = "5")]
pub(super) body: Option<AnyValue>,
#[prost(message, repeated, tag = "6")]
pub(super) attributes: Vec<KeyValue>,
#[prost(uint32, tag = "7")]
pub(super) dropped_attributes_count: u32,
#[prost(fixed32, tag = "8")]
pub(super) flags: u32,
#[prost(bytes = "vec", tag = "9")]
pub(super) trace_id: Vec<u8>,
#[prost(bytes = "vec", tag = "10")]
pub(super) span_id: Vec<u8>,
#[prost(fixed64, tag = "11")]
pub(super) observed_time_unix_nano: u64,
}
#[derive(Clone, PartialEq, ::prost::Message)]
pub(super) struct Resource {
#[prost(message, repeated, tag = "1")]
pub(super) attributes: Vec<KeyValue>,
#[prost(uint32, tag = "2")]
pub(super) dropped_attributes_count: u32,
}
#[derive(Clone, PartialEq, ::prost::Message)]
pub(super) struct InstrumentationScope {
#[prost(string, tag = "1")]
pub(super) name: String,
#[prost(string, tag = "2")]
pub(super) version: String,
#[prost(message, repeated, tag = "3")]
pub(super) attributes: Vec<KeyValue>,
#[prost(uint32, tag = "4")]
pub(super) dropped_attributes_count: u32,
}
#[derive(Clone, PartialEq, ::prost::Message)]
pub(super) struct KeyValue {
#[prost(string, tag = "1")]
pub(super) key: String,
#[prost(message, optional, tag = "2")]
pub(super) value: Option<AnyValue>,
}
#[derive(Clone, PartialEq, ::prost::Message)]
pub(super) struct AnyValue {
#[prost(oneof = "AnyValueKind", tags = "1, 2, 3, 4, 5, 6, 7")]
pub(super) value: Option<AnyValueKind>,
}
#[allow(clippy::enum_variant_names)]
#[derive(Clone, PartialEq, ::prost::Oneof)]
pub(super) enum AnyValueKind {
#[prost(string, tag = "1")]
StringValue(String),
#[prost(bool, tag = "2")]
BoolValue(bool),
#[prost(int64, tag = "3")]
IntValue(i64),
#[prost(double, tag = "4")]
DoubleValue(f64),
#[prost(message, tag = "5")]
ArrayValue(ArrayValue),
#[prost(message, tag = "6")]
KvlistValue(KeyValueList),
#[prost(bytes = "vec", tag = "7")]
BytesValue(Vec<u8>),
}
#[derive(Clone, PartialEq, ::prost::Message)]
pub(super) struct ArrayValue {
#[prost(message, repeated, tag = "1")]
pub(super) values: Vec<AnyValue>,
}
#[derive(Clone, PartialEq, ::prost::Message)]
pub(super) struct KeyValueList {
#[prost(message, repeated, tag = "1")]
pub(super) values: Vec<KeyValue>,
}
}
fn decode_request(bytes: &[u8]) -> otlp::ExportLogsServiceRequest {
<otlp::ExportLogsServiceRequest as prost::Message>::decode(bytes)
.expect("the emitted bytes must parse as an OTLP ExportLogsServiceRequest")
}
fn decode_record(bytes: &[u8]) -> otlp::LogRecord {
let request = decode_request(bytes);
let [resource_logs] = &request.resource_logs[..] else {
panic!("exactly one ResourceLogs");
};
let [scope_logs] = &resource_logs.scope_logs[..] else {
panic!("exactly one ScopeLogs");
};
let [record] = &scope_logs.log_records[..] else {
panic!("exactly one LogRecord");
};
record.clone()
}
fn any(value: &Option<otlp::AnyValue>) -> Option<otlp::AnyValueKind> {
value.as_ref().and_then(|v| v.value.clone())
}
fn padded_entry(len: usize) -> LogEntry {
LogEntry {
time_unix_nano: 1_755_000_000_000_000_000,
severity: Severity::Info,
body: "x".repeat(len),
attrs: vec![("bugwarden.pad", AttrValue::Str("y".repeat(len)))],
trace: None,
}
}
#[test]
fn varints_are_byte_for_byte_what_prost_writes_and_reads() {
let mut values = vec![0u64, 300, u64::MAX];
for shift in 0..64 {
values.push(1u64 << shift);
values.push((1u64 << shift) - 1);
}
for value in values {
let mut ours = Vec::new();
wire::put_varint(&mut ours, value);
let mut theirs = Vec::new();
prost::encoding::encode_varint(value, &mut theirs);
assert_eq!(ours, theirs, "varint encoding of {value}");
let mut cursor = ours.as_slice();
assert_eq!(
prost::encoding::decode_varint(&mut cursor).ok(),
Some(value),
"prost must read back {value}"
);
assert!(cursor.is_empty(), "{value} must use no spare bytes");
}
}
#[test]
fn an_independent_decoder_reads_back_the_whole_request() {
let entry = audit_entry(
&event(tool_call(Some(TraceContext {
trace_id: "4bf92f3577b34da6a3ce929d0e0e4736".to_owned(),
span_id: "00f067aa0ba902b7".to_owned(),
}))),
br#"{"seq":42}"#,
);
let bytes = encode_request("bugwarden-edge", std::slice::from_ref(&entry));
let request = decode_request(&bytes);
let [resource_logs] = &request.resource_logs[..] else {
panic!("exactly one ResourceLogs");
};
let resource = resource_logs.resource.as_ref().expect("a Resource");
let [service] = &resource.attributes[..] else {
panic!("exactly one resource attribute");
};
assert_eq!(service.key, "service.name");
assert_eq!(
any(&service.value),
Some(otlp::AnyValueKind::StringValue("bugwarden-edge".to_owned()))
);
let [scope_logs] = &resource_logs.scope_logs[..] else {
panic!("exactly one ScopeLogs");
};
let scope = scope_logs.scope.as_ref().expect("an InstrumentationScope");
assert_eq!(scope.name, SCOPE_NAME);
assert_eq!(scope.version, env!("CARGO_PKG_VERSION"));
let [record] = &scope_logs.log_records[..] else {
panic!("exactly one LogRecord");
};
assert_eq!(record.severity_number, i32::from(Severity::Info as u8));
assert_eq!(record.severity_text, "INFO");
assert_eq!(
any(&record.body),
Some(otlp::AnyValueKind::StringValue(r#"{"seq":42}"#.to_owned()))
);
assert_eq!(record.time_unix_nano, entry.time_unix_nano);
assert_eq!(record.observed_time_unix_nano, entry.time_unix_nano);
assert_eq!(record.trace_id.len(), 16);
assert_eq!(record.span_id.len(), 8);
assert_eq!(record.trace_id[0], 0x4b);
assert_eq!(record.span_id[1], 0xf0);
let decoded: Vec<(&str, otlp::AnyValueKind)> = record
.attributes
.iter()
.map(|kv| {
(
kv.key.as_str(),
any(&kv.value).expect("an attribute must carry a value"),
)
})
.collect();
let expected: Vec<(&str, otlp::AnyValueKind)> = entry
.attrs
.iter()
.map(|(key, value)| {
let value = match value {
AttrValue::Str(s) => otlp::AnyValueKind::StringValue(s.clone()),
AttrValue::Int(i) => otlp::AnyValueKind::IntValue(*i),
};
(*key, value)
})
.collect();
assert!(!expected.is_empty(), "the fixture must carry attributes");
assert_eq!(decoded, expected);
}
#[test]
fn re_encoding_the_decoded_request_reproduces_every_byte() {
let entries = vec![
audit_entry(&event(tool_call(None)), b"{}"),
audit_entry(
&event(tool_call(Some(TraceContext {
trace_id: "4bf92f3577b34da6a3ce929d0e0e4736".to_owned(),
span_id: "00f067aa0ba902b7".to_owned(),
}))),
br#"{"seq":42}"#,
),
audit_entry(
&event(AuditEventKind::AuditGap(crate::audit::AuditGapEvent {
dropped: 2,
reason: crate::audit::GapReason::WriteError,
})),
b"{}",
),
];
let bytes = encode_request("bugwarden", &entries);
assert_eq!(
prost::Message::encode_to_vec(&decode_request(&bytes)),
bytes,
"the request must contain nothing but the fields OTLP declares"
);
}
#[test]
fn length_prefixes_hold_on_both_sides_of_every_boundary() {
for len in [
0usize, 1, 126, 127, 128, 129, 16_382, 16_383, 16_384, 16_385,
] {
let entry = padded_entry(len);
let bytes = encode_request("bugwarden", std::slice::from_ref(&entry));
assert_eq!(
any(&decode_record(&bytes).body),
Some(otlp::AnyValueKind::StringValue(entry.body.clone())),
"a body of {len} bytes"
);
assert_eq!(
prost::Message::encode_to_vec(&decode_request(&bytes)),
bytes,
"a body of {len} bytes"
);
}
}
#[test]
fn attribute_integers_survive_the_signed_varint_extremes() {
for value in [0i64, 1, -1, 127, 128, -128, i64::MAX, i64::MIN] {
let mut entry = padded_entry(0);
entry.attrs = vec![("bugwarden.response_bytes", AttrValue::Int(value))];
let bytes = encode_request("bugwarden", std::slice::from_ref(&entry));
let record = decode_record(&bytes);
let [attribute] = &record.attributes[..] else {
panic!("exactly one attribute");
};
assert_eq!(
any(&attribute.value),
Some(otlp::AnyValueKind::IntValue(value)),
"{value} must survive the wire"
);
assert_eq!(
prost::Message::encode_to_vec(&decode_request(&bytes)),
bytes,
"{value}"
);
}
}
#[test]
fn drops_are_logged_at_powers_of_two_only() {
let dropped = AtomicU64::new(0);
let threshold = AtomicU64::new(1);
let (_, logs) = capture_logs(|| {
for _ in 0..16 {
note_drops(&dropped, &threshold, 1, REASON_QUEUE_FULL);
}
});
assert_eq!(dropped.load(Ordering::Relaxed), 16);
let lines = logs
.as_str()
.matches("otlp export is dropping diagnostic records")
.count();
assert_eq!(
lines,
5,
"one line per crossing of 1, 2, 4, 8, 16 — not one per record: {}",
logs.as_str()
);
assert_logged(&logs, "dropped=16");
}
#[test]
fn a_batch_sized_drop_logs_once_and_moves_the_threshold_past_it() {
let dropped = AtomicU64::new(0);
let threshold = AtomicU64::new(1);
let (_, logs) = capture_logs(|| {
note_drops(&dropped, &threshold, 300, REASON_NETWORK);
note_drops(&dropped, &threshold, 1, REASON_NETWORK);
});
assert_eq!(
logs.as_str()
.matches("otlp export is dropping diagnostic records")
.count(),
1,
"a 300-record batch crosses nine powers of two (1..256) and still logs once: {}",
logs.as_str()
);
assert_eq!(threshold.load(Ordering::Relaxed), 512);
}
#[test]
fn the_drop_line_names_no_endpoint_and_no_header() {
let dropped = AtomicU64::new(0);
let threshold = AtomicU64::new(1);
let (_, logs) = capture_logs(|| {
note_drops(&dropped, &threshold, 1, REASON_NETWORK);
});
assert_logged(&logs, "reason=\"network\"");
for forbidden in ["http://", "https://", "authorization", "Bearer", "4318"] {
logs.assert_not_contains(forbidden);
}
}
#[tokio::test]
async fn an_unreachable_collector_loses_audit_records_loudly_not_as_drops() {
let cfg = resolve(&env(&refused_base_url()))
.expect("resolves")
.expect("export is on");
let pipeline = Arc::new(Pipeline::start(cfg).expect("the pipeline must start"));
assert!(
!pipeline.delivery_failing(),
"before any attempt, delivery is not known to be failing"
);
pipeline
.accept(&event(tool_call(None)), b"{}")
.expect("an open queue takes custody");
tokio::time::sleep(BATCH_INTERVAL * 3).await;
assert!(
pipeline.delivery_failing(),
"a failed delivery must mark the pipeline failing"
);
assert_eq!(
pipeline.dropped(),
0,
"audit records never ride the diagnostics drop counter"
);
assert_eq!(
pipeline.take_lost(),
1,
"the undelivered record must be surfaced for gap accounting"
);
assert_eq!(pipeline.take_lost(), 0, "take_lost drains the count");
pipeline.shutdown().await;
}
#[tokio::test]
async fn an_unreachable_collector_counts_diagnostics_as_drops() {
let cfg = resolve(&env(&refused_base_url()))
.expect("resolves")
.expect("export is on");
let pipeline = Arc::new(Pipeline::start(cfg).expect("the pipeline must start"));
pipeline.emit(LogEntry {
time_unix_nano: now_unix_nano(),
severity: Severity::Info,
body: "diagnostic".to_owned(),
attrs: Vec::new(),
trace: None,
});
tokio::time::sleep(BATCH_INTERVAL * 3).await;
pipeline.shutdown().await;
assert!(
pipeline.dropped() >= 1,
"an unreachable collector must count the diagnostics it lost"
);
assert_eq!(
pipeline.take_lost(),
0,
"a dropped diagnostic is not an audit loss"
);
}
#[tokio::test]
async fn a_record_offered_after_shutdown_is_refused_not_dropped() {
let cfg = resolve(&env(NO_COLLECTOR))
.expect("resolves")
.expect("export is on");
let pipeline = Arc::new(Pipeline::start(cfg).expect("the pipeline must start"));
pipeline.shutdown().await;
let before = pipeline.dropped();
assert!(
pipeline.accept(&event(tool_call(None)), b"{}").is_err(),
"a shut-down pipeline must refuse custody"
);
assert!(
pipeline.delivery_failing(),
"a refused record marks delivery failing"
);
assert_eq!(
pipeline.dropped(),
before,
"a refused audit record must not be counted as a drop"
);
}
#[tokio::test]
async fn a_diagnostic_offered_after_shutdown_is_a_shutdown_drop() {
let cfg = resolve(&env(NO_COLLECTOR))
.expect("resolves")
.expect("export is on");
let pipeline = Arc::new(Pipeline::start(cfg).expect("the pipeline must start"));
pipeline.shutdown().await;
let before = pipeline.dropped();
let (_, logs) = capture_logs(|| {
pipeline.emit(LogEntry {
time_unix_nano: now_unix_nano(),
severity: Severity::Info,
body: "late diagnostic".to_owned(),
attrs: Vec::new(),
trace: None,
});
});
assert_eq!(
pipeline.dropped(),
before + 1,
"a diagnostic offered to a shut-down pipeline is still counted"
);
assert_logged(&logs, "reason=\"shutdown\"");
}
#[test]
fn delivery_notes_log_transitions_only() {
let healthy = AtomicBool::new(true);
let (_, logs) = capture_logs(|| {
note_delivery(&healthy, false);
note_delivery(&healthy, false);
note_delivery(&healthy, false);
});
assert_eq!(
logs.as_str().matches("otlp delivery is failing").count(),
1,
"repeated failures log once: {}",
logs.as_str()
);
assert!(!healthy.load(Ordering::Relaxed));
let (_, logs) = capture_logs(|| {
note_delivery(&healthy, true);
note_delivery(&healthy, true);
});
assert_eq!(
logs.as_str().matches("otlp delivery recovered").count(),
1,
"repeated successes log once: {}",
logs.as_str()
);
assert!(healthy.load(Ordering::Relaxed));
}
#[tokio::test]
async fn the_startup_probe_refuses_a_dead_collector_without_naming_it() {
let (base, host, port) = dead_collector();
let cfg = resolve(&env(&base))
.expect("resolves")
.expect("export is on");
let pipeline = Arc::new(Pipeline::start(cfg).expect("the pipeline must start"));
let err = pipeline
.probe()
.await
.expect_err("a dead collector must refuse startup");
let message = format!("{err}");
assert!(
message.contains(ENDPOINT_VAR) && message.contains(HEADERS_VAR),
"the refusal must point at the configuration: {message}"
);
assert!(
!message.contains(LOGS_ENDPOINT_VAR) && !message.contains(LOGS_HEADERS_VAR),
"a general-variable config must not blame the logs-specific pair: {message}"
);
assert!(
!message.contains(&host) && !message.contains(&port),
"the refusal must not carry the endpoint: {message}"
);
assert!(
pipeline.delivery_failing(),
"a failed probe leaves delivery marked failing"
);
pipeline.shutdown().await;
}
#[tokio::test]
async fn the_startup_probe_names_the_logs_specific_variables_when_those_won() {
let (base, host, port) = dead_collector();
let cfg = resolve(&OtelEnv {
logs_endpoint: Some(format!("{base}/v1/logs")),
logs_headers: Some("authorization=Bearer x".to_owned()),
..OtelEnv::default()
})
.expect("resolves")
.expect("a logs-specific endpoint turns export on");
let pipeline = Arc::new(Pipeline::start(cfg).expect("the pipeline must start"));
let message = format!(
"{}",
pipeline
.probe()
.await
.expect_err("a dead collector must refuse startup")
);
assert!(
message.contains(LOGS_ENDPOINT_VAR) && message.contains(LOGS_HEADERS_VAR),
"the refusal must name the variables that actually won: {message}"
);
assert!(
!message.contains(&host) && !message.contains(&port) && !message.contains("Bearer"),
"the refusal must not carry the endpoint or the credential: {message}"
);
pipeline.shutdown().await;
}
#[tokio::test]
async fn shutdown_is_idempotent_and_bounded() {
let cfg = resolve(&env(NO_COLLECTOR))
.expect("resolves")
.expect("export is on");
let pipeline = Arc::new(Pipeline::start(cfg).expect("the pipeline must start"));
pipeline.shutdown().await;
pipeline.shutdown().await;
}
#[test]
fn the_pipeline_debug_carries_no_configuration() {
let rendered = {
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("a runtime");
let _guard = runtime.enter();
let cfg = resolve(&OtelEnv {
headers: Some("authorization=Bearer super-secret-token".to_owned()),
..env("http://collector.example:4318")
})
.expect("resolves")
.expect("export is on");
format!(
"{:?}",
Pipeline::start(cfg).expect("the pipeline must start")
)
};
for forbidden in [
"collector.example",
"super-secret-token",
"authorization",
"4318",
] {
assert!(
!rendered.contains(forbidden),
"Pipeline's Debug must not carry {forbidden}: {rendered}"
);
}
}
}