use std::collections::VecDeque;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::time::SystemTime;
use opentelemetry::trace::{Event, SpanId, SpanKind, Status, TraceId};
use opentelemetry::{Array, KeyValue, Value};
use parking_lot::Mutex;
use crate::observe::{FilterSpan, TelemetrySink};
pub const DEFAULT_QUEUE_CAPACITY: usize = 2048;
#[derive(Debug, Clone)]
pub struct SpanRecord {
pub trace_id: TraceId,
pub span_id: SpanId,
pub parent_span_id: Option<SpanId>,
pub name: String,
pub kind: SpanKind,
pub start_time: SystemTime,
pub end_time: SystemTime,
pub status: Status,
pub attributes: Vec<KeyValue>,
pub events: Vec<Event>,
pub remote_parent: bool,
pub sampled: bool,
}
impl SpanRecord {
#[allow(clippy::too_many_arguments)]
pub fn request_span(
trace: &crate::observe::RequestTrace,
method: &str,
path: &str,
scheme: &str,
status_code: u16,
start_time: SystemTime,
duration: std::time::Duration,
) -> Self {
let status = if status_code >= 500 {
Status::Error {
description: std::borrow::Cow::Borrowed(""),
}
} else {
Status::Unset
};
Self {
trace_id: trace.trace_id(),
span_id: trace.request_span_id(),
parent_span_id: trace.parent_span_id(),
name: method.to_string(),
kind: SpanKind::Server,
start_time,
end_time: start_time + duration,
status,
attributes: vec![
KeyValue::new("http.request.method", method.to_string()),
KeyValue::new("url.path", path.to_string()),
KeyValue::new("url.scheme", scheme.to_string()),
KeyValue::new("http.response.status_code", i64::from(status_code)),
],
events: vec![],
remote_parent: trace.parent_span_id().is_some(),
sampled: trace.is_sampled(),
}
}
}
impl From<&FilterSpan> for SpanRecord {
fn from(span: &FilterSpan) -> Self {
Self {
trace_id: span.trace_id,
span_id: span.span_id,
parent_span_id: Some(span.parent_span_id),
name: span.name.clone(),
kind: span.kind.clone(),
start_time: span.start_time,
end_time: span.start_time + span.duration,
status: span.status(),
attributes: span.attributes.clone(),
events: span.events.clone(),
remote_parent: false,
sampled: span.sampled,
}
}
}
pub struct OtlpBuffer {
queue: Mutex<VecDeque<SpanRecord>>,
capacity: usize,
dropped: AtomicU64,
drop_logged: AtomicBool,
}
impl OtlpBuffer {
pub fn new(capacity: usize) -> Self {
Self {
queue: Mutex::new(VecDeque::with_capacity(capacity)),
capacity,
dropped: AtomicU64::new(0),
drop_logged: AtomicBool::new(false),
}
}
pub fn push(&self, record: SpanRecord) {
{
let mut queue = self.queue.lock();
if queue.len() < self.capacity {
queue.push_back(record);
return;
}
}
self.dropped.fetch_add(1, Ordering::Relaxed);
if !self.drop_logged.swap(true, Ordering::Relaxed) {
tracing::warn!(
capacity = self.capacity,
"OTLP span queue full; dropping spans (further drops are counted, not logged)"
);
}
}
pub fn drain(&self, max: usize) -> Vec<SpanRecord> {
let mut queue = self.queue.lock();
let n = queue.len().min(max);
queue.drain(..n).collect()
}
pub fn len(&self) -> usize {
self.queue.lock().len()
}
pub fn is_empty(&self) -> bool {
self.queue.lock().is_empty()
}
pub fn dropped_spans(&self) -> u64 {
self.dropped.load(Ordering::Relaxed)
}
pub fn record_dropped(&self, n: u64) {
self.dropped.fetch_add(n, Ordering::Relaxed);
}
}
impl Default for OtlpBuffer {
fn default() -> Self {
Self::new(DEFAULT_QUEUE_CAPACITY)
}
}
impl TelemetrySink for OtlpBuffer {
fn export(&self, span: &FilterSpan) {
if !span.sampled {
return;
}
self.push(span.into());
}
}
const WIRE_VARINT: u32 = 0;
const WIRE_I64: u32 = 1;
const WIRE_LEN: u32 = 2;
const WIRE_I32: u32 = 5;
fn put_varint(buf: &mut Vec<u8>, mut v: u64) {
loop {
let byte = (v & 0x7f) as u8;
v >>= 7;
if v == 0 {
buf.push(byte);
return;
}
buf.push(byte | 0x80);
}
}
fn put_tag(buf: &mut Vec<u8>, field: u32, wire: u32) {
put_varint(buf, u64::from((field << 3) | wire));
}
fn put_len(buf: &mut Vec<u8>, field: u32, bytes: &[u8]) {
put_tag(buf, field, WIRE_LEN);
put_varint(buf, bytes.len() as u64);
buf.extend_from_slice(bytes);
}
fn put_str(buf: &mut Vec<u8>, field: u32, s: &str) {
if !s.is_empty() {
put_len(buf, field, s.as_bytes());
}
}
fn put_u64(buf: &mut Vec<u8>, field: u32, v: u64) {
if v != 0 {
put_tag(buf, field, WIRE_VARINT);
put_varint(buf, v);
}
}
fn put_fixed64(buf: &mut Vec<u8>, field: u32, v: u64) {
if v != 0 {
put_tag(buf, field, WIRE_I64);
buf.extend_from_slice(&v.to_le_bytes());
}
}
fn put_fixed32(buf: &mut Vec<u8>, field: u32, v: u32) {
if v != 0 {
put_tag(buf, field, WIRE_I32);
buf.extend_from_slice(&v.to_le_bytes());
}
}
fn unix_nanos(t: SystemTime) -> u64 {
t.duration_since(SystemTime::UNIX_EPOCH)
.map(|d| d.as_nanos() as u64)
.unwrap_or(0)
}
fn encode_any_value(value: &Value) -> Vec<u8> {
let mut buf = Vec::new();
match value {
Value::String(s) => put_len(&mut buf, 1, s.as_str().as_bytes()),
Value::Bool(b) => {
put_tag(&mut buf, 2, WIRE_VARINT);
put_varint(&mut buf, u64::from(*b));
}
Value::I64(i) => {
put_tag(&mut buf, 3, WIRE_VARINT);
put_varint(&mut buf, *i as u64);
}
Value::F64(f) => {
put_tag(&mut buf, 4, WIRE_I64);
buf.extend_from_slice(&f.to_bits().to_le_bytes());
}
Value::Array(array) => {
let mut inner = Vec::new();
let one = |v: Value| encode_any_value(&v);
match array {
Array::Bool(items) => {
for b in items {
put_len(&mut inner, 1, &one(Value::Bool(*b)));
}
}
Array::I64(items) => {
for i in items {
put_len(&mut inner, 1, &one(Value::I64(*i)));
}
}
Array::F64(items) => {
for f in items {
put_len(&mut inner, 1, &one(Value::F64(*f)));
}
}
Array::String(items) => {
for s in items {
put_len(&mut inner, 1, &one(Value::String(s.clone())));
}
}
_ => {}
}
put_len(&mut buf, 5, &inner);
}
other => put_len(&mut buf, 1, format!("{other:?}").as_bytes()),
}
buf
}
fn encode_key_value(kv: &KeyValue) -> Vec<u8> {
let mut buf = Vec::new();
put_str(&mut buf, 1, kv.key.as_str());
put_len(&mut buf, 2, &encode_any_value(&kv.value));
buf
}
fn encode_event(event: &Event) -> Vec<u8> {
let mut buf = Vec::new();
put_fixed64(&mut buf, 1, unix_nanos(event.timestamp));
put_str(&mut buf, 2, &event.name);
for kv in &event.attributes {
put_len(&mut buf, 3, &encode_key_value(kv));
}
buf
}
fn kind_value(kind: &SpanKind) -> u64 {
match kind {
SpanKind::Internal => 1,
SpanKind::Server => 2,
SpanKind::Client => 3,
SpanKind::Producer => 4,
SpanKind::Consumer => 5,
}
}
fn span_flags(record: &SpanRecord) -> u32 {
let sampled = u32::from(record.sampled);
let remote = if record.remote_parent { 0x200 } else { 0 };
0x100 | remote | sampled
}
fn encode_status(status: &Status) -> Option<Vec<u8>> {
let mut buf = Vec::new();
match status {
Status::Unset => return None,
Status::Ok => put_u64(&mut buf, 3, 1),
Status::Error { description } => {
put_str(&mut buf, 2, description);
put_u64(&mut buf, 3, 2);
}
}
Some(buf)
}
fn encode_span(record: &SpanRecord) -> Vec<u8> {
let mut buf = Vec::new();
put_len(&mut buf, 1, &record.trace_id.to_bytes());
put_len(&mut buf, 2, &record.span_id.to_bytes());
if let Some(parent) = record.parent_span_id {
put_len(&mut buf, 4, &parent.to_bytes());
}
put_str(&mut buf, 5, &record.name);
put_u64(&mut buf, 6, kind_value(&record.kind));
put_fixed64(&mut buf, 7, unix_nanos(record.start_time));
put_fixed64(&mut buf, 8, unix_nanos(record.end_time));
for kv in &record.attributes {
put_len(&mut buf, 9, &encode_key_value(kv));
}
for event in &record.events {
put_len(&mut buf, 11, &encode_event(event));
}
if let Some(status) = encode_status(&record.status) {
put_len(&mut buf, 15, &status);
}
put_fixed32(&mut buf, 16, span_flags(record));
buf
}
fn string_attr(key: &str, value: &str) -> Vec<u8> {
encode_key_value(&KeyValue::new(key.to_string(), value.to_string()))
}
pub fn encode_traces_request(service_name: &str, spans: &[SpanRecord]) -> Vec<u8> {
let mut resource = Vec::new();
put_len(&mut resource, 1, &string_attr("service.name", service_name));
put_len(
&mut resource,
1,
&string_attr("telemetry.sdk.name", "plecto"),
);
put_len(
&mut resource,
1,
&string_attr("telemetry.sdk.language", "rust"),
);
put_len(
&mut resource,
1,
&string_attr("telemetry.sdk.version", env!("CARGO_PKG_VERSION")),
);
let mut scope = Vec::new();
put_str(&mut scope, 1, "plecto");
put_str(&mut scope, 2, env!("CARGO_PKG_VERSION"));
let mut scope_spans = Vec::new();
put_len(&mut scope_spans, 1, &scope);
for record in spans {
put_len(&mut scope_spans, 2, &encode_span(record));
}
let mut resource_spans = Vec::new();
put_len(&mut resource_spans, 1, &resource);
put_len(&mut resource_spans, 2, &scope_spans);
let mut request = Vec::new();
put_len(&mut request, 1, &resource_spans);
request
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct PartialSuccess {
pub rejected_spans: i64,
pub error_message: String,
}
pub fn decode_export_partial_success(body: &[u8]) -> Option<PartialSuccess> {
let inner = read_fields(body, |field, payload| match (field, payload) {
(1, FieldPayload::Len(bytes)) => Some(bytes.to_vec()),
_ => None,
})?;
let inner = inner?;
let mut rejected: i64 = 0;
let mut message = String::new();
read_fields(&inner, |field, payload| {
match (field, payload) {
(1, FieldPayload::Varint(v)) => rejected = v as i64,
(2, FieldPayload::Len(bytes)) => {
message = String::from_utf8_lossy(bytes).into_owned();
}
_ => {}
}
None::<()>
})?;
if rejected == 0 && message.is_empty() {
return None;
}
Some(PartialSuccess {
rejected_spans: rejected,
error_message: message,
})
}
enum FieldPayload<'a> {
Varint(u64),
Len(&'a [u8]),
}
fn read_fields<'a, T>(
mut bytes: &'a [u8],
mut visit: impl FnMut(u32, FieldPayload<'a>) -> Option<T>,
) -> Option<Option<T>> {
while !bytes.is_empty() {
let (tag, rest) = read_varint(bytes)?;
let field = (tag >> 3) as u32;
let wire = (tag & 0x7) as u32;
bytes = rest;
let payload = match wire {
WIRE_VARINT => {
let (v, rest) = read_varint(bytes)?;
bytes = rest;
FieldPayload::Varint(v)
}
WIRE_I64 => {
let v = u64::from_le_bytes(bytes.get(..8)?.try_into().ok()?);
bytes = bytes.get(8..)?;
FieldPayload::Varint(v)
}
WIRE_LEN => {
let (len, rest) = read_varint(bytes)?;
let len = usize::try_from(len).ok()?;
let payload = rest.get(..len)?;
bytes = rest.get(len..)?;
FieldPayload::Len(payload)
}
WIRE_I32 => {
let v = u32::from_le_bytes(bytes.get(..4)?.try_into().ok()?);
bytes = bytes.get(4..)?;
FieldPayload::Varint(u64::from(v))
}
_ => return None,
};
if let Some(t) = visit(field, payload) {
return Some(Some(t));
}
}
Some(None)
}
fn read_varint(bytes: &[u8]) -> Option<(u64, &[u8])> {
let mut value: u64 = 0;
for (i, byte) in bytes.iter().enumerate().take(10) {
value |= u64::from(byte & 0x7f) << (7 * i);
if byte & 0x80 == 0 {
return Some((value, bytes.get(i + 1..)?));
}
}
None
}
#[cfg(test)]
mod tests {
use std::time::Duration;
use opentelemetry_proto::tonic::collector::trace::v1::{
ExportTracePartialSuccess, ExportTraceServiceRequest, ExportTraceServiceResponse,
};
use opentelemetry_proto::tonic::common::v1::any_value::Value as ProtoValue;
use opentelemetry_proto::tonic::trace::v1::status::StatusCode as ProtoStatusCode;
use prost::Message;
use super::*;
use crate::observe::{Hook, RequestTrace, SpanOutcome, build_filter_span};
use crate::{Isolation, LogLevel, LogLine};
fn record(name: &str) -> SpanRecord {
let start = SystemTime::UNIX_EPOCH + Duration::from_secs(1_700_000_000);
SpanRecord {
trace_id: TraceId::from_bytes([1; 16]),
span_id: SpanId::from_bytes([2; 8]),
parent_span_id: None,
name: name.to_string(),
kind: SpanKind::Server,
start_time: start,
end_time: start + Duration::from_millis(5),
status: Status::Unset,
attributes: vec![],
events: vec![],
remote_parent: false,
sampled: true,
}
}
fn decode(bytes: &[u8]) -> ExportTraceServiceRequest {
ExportTraceServiceRequest::decode(bytes).expect("official decoder accepts our bytes")
}
#[test]
fn round_trips_a_full_span_through_the_official_decoder() {
let start = SystemTime::UNIX_EPOCH + Duration::from_secs(1_700_000_000);
let rec = SpanRecord {
trace_id: TraceId::from_bytes([0xAB; 16]),
span_id: SpanId::from_bytes([0xCD; 8]),
parent_span_id: Some(SpanId::from_bytes([0xEF; 8])),
name: "GET".to_string(),
kind: SpanKind::Server,
start_time: start,
end_time: start + Duration::from_millis(12),
status: Status::Error {
description: "upstream 502".into(),
},
attributes: vec![
KeyValue::new("http.request.method", "GET"),
KeyValue::new("http.response.status_code", 502_i64),
KeyValue::new("plecto.pi", 3.5_f64),
KeyValue::new("plecto.flag", true),
KeyValue::new(
"plecto.list",
Value::Array(Array::String(vec!["a".into(), "b".into()])),
),
],
events: vec![Event::new(
"auth denied",
start,
vec![KeyValue::new("log.level", "warn")],
0,
)],
remote_parent: true,
sampled: true,
};
let decoded = decode(&encode_traces_request("plecto", &[rec]));
let resource = decoded.resource_spans[0].resource.as_ref().unwrap();
let attr = |key: &str| {
resource
.attributes
.iter()
.find(|kv| kv.key == key)
.and_then(|kv| kv.value.as_ref())
.and_then(|v| v.value.as_ref())
.cloned()
};
assert_eq!(
attr("service.name"),
Some(ProtoValue::StringValue("plecto".into()))
);
assert_eq!(
attr("telemetry.sdk.name"),
Some(ProtoValue::StringValue("plecto".into())),
"a hand-written exporter must not claim to be the official SDK"
);
let scope_spans = &decoded.resource_spans[0].scope_spans[0];
assert_eq!(scope_spans.scope.as_ref().unwrap().name, "plecto");
let span = &scope_spans.spans[0];
assert_eq!(span.trace_id, vec![0xAB; 16]);
assert_eq!(span.span_id, vec![0xCD; 8]);
assert_eq!(span.parent_span_id, vec![0xEF; 8]);
assert_eq!(span.name, "GET");
assert_eq!(span.kind, 2, "SERVER");
assert_eq!(span.start_time_unix_nano, 1_700_000_000_000_000_000);
assert_eq!(span.end_time_unix_nano, 1_700_000_000_012_000_000);
assert_eq!(
span.flags,
0x300 | 0x01,
"remote parent: has_is_remote + is_remote + sampled"
);
let span_attr = |key: &str| {
span.attributes
.iter()
.find(|kv| kv.key == key)
.and_then(|kv| kv.value.as_ref())
.and_then(|v| v.value.as_ref())
.cloned()
};
assert_eq!(
span_attr("http.request.method"),
Some(ProtoValue::StringValue("GET".into()))
);
assert_eq!(
span_attr("http.response.status_code"),
Some(ProtoValue::IntValue(502))
);
assert_eq!(span_attr("plecto.pi"), Some(ProtoValue::DoubleValue(3.5)));
assert_eq!(span_attr("plecto.flag"), Some(ProtoValue::BoolValue(true)));
match span_attr("plecto.list") {
Some(ProtoValue::ArrayValue(arr)) => {
let items: Vec<_> = arr
.values
.iter()
.filter_map(|v| v.value.as_ref())
.cloned()
.collect();
assert_eq!(
items,
vec![
ProtoValue::StringValue("a".into()),
ProtoValue::StringValue("b".into())
]
);
}
other => panic!("expected an array attribute, got {other:?}"),
}
let status = span.status.as_ref().expect("error status is encoded");
assert_eq!(status.code, ProtoStatusCode::Error as i32);
assert_eq!(status.message, "upstream 502");
assert_eq!(span.events.len(), 1);
assert_eq!(span.events[0].name, "auth denied");
assert_eq!(span.events[0].time_unix_nano, 1_700_000_000_000_000_000);
}
#[test]
fn root_span_omits_parent_and_unset_status_and_clears_remote_bit() {
let decoded = decode(&encode_traces_request("plecto", &[record("root")]));
let span = &decoded.resource_spans[0].scope_spans[0].spans[0];
assert!(span.parent_span_id.is_empty(), "no parent field for a root");
assert!(span.status.is_none(), "Unset status is omitted entirely");
assert_eq!(span.flags, 0x100 | 0x01, "local parent context, sampled");
}
#[test]
fn filter_span_converts_with_local_parent_and_derived_end_time() {
let trace = RequestTrace::root();
let logs = vec![LogLine {
level: LogLevel::Info,
message: "hi".to_string(),
}];
let start = SystemTime::now();
let span = build_filter_span(
&trace,
"auth",
Isolation::Trusted,
Hook::OnRequest,
SpanOutcome::Continue,
start,
Duration::from_micros(250),
&logs,
);
let rec = SpanRecord::from(&span);
assert_eq!(rec.parent_span_id, Some(trace.request_span_id()));
assert!(!rec.remote_parent, "a filter span's parent is always local");
assert_eq!(rec.end_time, start + Duration::from_micros(250));
assert_eq!(rec.status, Status::Ok);
assert_eq!(rec.events.len(), 1);
let decoded = decode(&encode_traces_request("plecto", &[rec]));
let got = &decoded.resource_spans[0].scope_spans[0].spans[0];
assert_eq!(got.kind, 1, "INTERNAL");
assert_eq!(got.name, "auth");
}
#[test]
fn buffer_is_fifo_bounded_and_counts_drops() {
let buffer = OtlpBuffer::new(2);
buffer.push(record("a"));
buffer.push(record("b"));
buffer.push(record("c")); assert_eq!(buffer.len(), 2);
assert_eq!(buffer.dropped_spans(), 1);
let first = buffer.drain(1);
assert_eq!(first.len(), 1);
assert_eq!(first[0].name, "a", "FIFO");
assert_eq!(buffer.drain(10).len(), 1);
assert!(buffer.is_empty());
buffer.record_dropped(3);
assert_eq!(buffer.dropped_spans(), 4, "export losses share the counter");
}
#[test]
fn sink_export_skips_unsampled_spans() {
let buffer = OtlpBuffer::default();
let trace = RequestTrace::from_traceparent(
"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00",
)
.expect("valid unsampled traceparent");
let span = build_filter_span(
&trace,
"f",
Isolation::Untrusted,
Hook::OnRequest,
SpanOutcome::Continue,
SystemTime::now(),
Duration::from_nanos(1),
&[],
);
buffer.export(&span);
assert!(buffer.is_empty(), "unsampled spans are not exported");
let sampled = build_filter_span(
&RequestTrace::root(),
"f",
Isolation::Untrusted,
Hook::OnRequest,
SpanOutcome::Continue,
SystemTime::now(),
Duration::from_nanos(1),
&[],
);
buffer.export(&sampled);
assert_eq!(buffer.len(), 1);
}
#[test]
fn decodes_partial_success_and_ignores_clean_responses() {
let with_rejection = ExportTraceServiceResponse {
partial_success: Some(ExportTracePartialSuccess {
rejected_spans: 7,
error_message: "bad spans".to_string(),
}),
}
.encode_to_vec();
assert_eq!(
decode_export_partial_success(&with_rejection),
Some(PartialSuccess {
rejected_spans: 7,
error_message: "bad spans".to_string(),
})
);
let clean = ExportTraceServiceResponse {
partial_success: None,
}
.encode_to_vec();
assert_eq!(decode_export_partial_success(&clean), None);
assert_eq!(decode_export_partial_success(b""), None);
assert_eq!(
decode_export_partial_success(&[0xff, 0xff, 0xff]),
None,
"malformed bytes are fail-soft"
);
}
}