use std::collections::{HashMap, HashSet};
use std::sync::Arc;
use opentelemetry_proto::tonic::collector::trace::v1::{
ExportTraceServiceRequest, ExportTraceServiceResponse,
};
use opentelemetry_proto::tonic::common::v1::{KeyValue, any_value};
use opentelemetry_proto::tonic::trace::v1::Span;
use tonic::{Request, Response, Status, async_trait};
use crate::event::{EventSource, EventType, SpanEvent};
use crate::report::metrics::{OtlpRejectReason, OtlpSpanFilterReason};
pub trait MetricsSink: Send + Sync {
fn record_otlp_reject(&self, reason: OtlpRejectReason);
fn record_otlp_spans(&self, stats: SpanConversionStats);
fn ingest_over_memory_limit(&self) -> bool {
false
}
}
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
pub struct SpanConversionStats {
pub received: u64,
pub filtered_not_io: u64,
pub filtered_missing_db_statement: u64,
pub filtered_missing_http_url: u64,
pub filtered_non_sql_datastore: u64,
pub filtered_merged_db_span: u64,
}
impl SpanConversionStats {
fn count_filtered(&mut self, reason: OtlpSpanFilterReason) {
match reason {
OtlpSpanFilterReason::NotIo => self.filtered_not_io += 1,
OtlpSpanFilterReason::MissingDbStatement => self.filtered_missing_db_statement += 1,
OtlpSpanFilterReason::MissingHttpUrl => self.filtered_missing_http_url += 1,
OtlpSpanFilterReason::NonSqlDatastore => self.filtered_non_sql_datastore += 1,
OtlpSpanFilterReason::MergedDbSpan => self.filtered_merged_db_span += 1,
}
}
#[must_use]
pub fn filtered_counts(&self) -> [(OtlpSpanFilterReason, u64); 5] {
[
(OtlpSpanFilterReason::NotIo, self.filtered_not_io),
(
OtlpSpanFilterReason::MissingDbStatement,
self.filtered_missing_db_statement,
),
(
OtlpSpanFilterReason::MissingHttpUrl,
self.filtered_missing_http_url,
),
(
OtlpSpanFilterReason::NonSqlDatastore,
self.filtered_non_sql_datastore,
),
(
OtlpSpanFilterReason::MergedDbSpan,
self.filtered_merged_db_span,
),
]
}
}
fn bytes_to_hex(bytes: &[u8]) -> String {
const HEX: &[u8; 16] = b"0123456789abcdef";
let mut out = String::with_capacity(bytes.len() * 2);
for &b in bytes {
out.push(HEX[(b >> 4) as usize] as char);
out.push(HEX[(b & 0x0f) as usize] as char);
}
out
}
use crate::time::nanos_to_iso8601;
#[inline]
fn any_value_as_str(value: Option<&any_value::Value>) -> Option<&str> {
match value {
Some(any_value::Value::StringValue(s)) => Some(s.as_str()),
_ => None,
}
}
#[inline]
fn any_value_as_int(value: Option<&any_value::Value>) -> Option<i64> {
match value {
Some(any_value::Value::IntValue(i)) => Some(*i),
_ => None,
}
}
fn get_str_attribute<'a>(attrs: &'a [KeyValue], key: &str) -> Option<&'a str> {
attrs
.iter()
.find(|kv| kv.key == key)
.and_then(|kv| any_value_as_str(kv.value.as_ref().and_then(|v| v.value.as_ref())))
}
use super::ANCESTOR_WALK_MAX_DEPTH as CODE_ATTRS_MAX_DEPTH;
const MAX_SPANS_PER_SERVICE: usize = 100_000;
const TRACE_ID_LEN: usize = 16;
type ServiceSpanIndexes<'a> = HashMap<&'a str, HashMap<&'a [u8], &'a Span>>;
type ServiceConsumerIndexes<'a> = HashMap<&'a str, HashMap<&'a [u8], Vec<&'a Span>>>;
#[derive(Default, Clone, Copy)]
struct CodeAttrs<'a> {
function_name: Option<&'a str>,
filepath: Option<&'a str>,
lineno: Option<i64>,
namespace: Option<&'a str>,
}
impl CodeAttrs<'_> {
#[inline]
fn has_any(&self) -> bool {
self.function_name.is_some()
|| self.filepath.is_some()
|| self.lineno.is_some()
|| self.namespace.is_some()
}
}
#[derive(Default)]
struct ClassifiedAttrs<'a> {
db_statement: Option<&'a str>,
db_query_text: Option<&'a str>,
db_system: Option<&'a str>,
db_system_name: Option<&'a str>,
dd_resource: Option<&'a str>,
db_type: Option<&'a str>,
http_url: Option<&'a str>,
url_full: Option<&'a str>,
http_method: Option<&'a str>,
http_request_method: Option<&'a str>,
rpc_system: Option<&'a str>,
rpc_service: Option<&'a str>,
rpc_method: Option<&'a str>,
messaging_system: Option<&'a str>,
messaging_destination_name: Option<&'a str>,
messaging_destination: Option<&'a str>,
messaging_body_size: Option<i64>,
http_status_code: Option<i64>,
http_response_status_code: Option<i64>,
http_response_body_size: Option<i64>,
http_response_content_length: Option<i64>,
cloud_region: Option<&'a str>,
code_function_name: Option<&'a str>,
code_function: Option<&'a str>,
code_file_path: Option<&'a str>,
code_filepath: Option<&'a str>,
code_line_number: Option<i64>,
code_lineno: Option<i64>,
code_namespace: Option<&'a str>,
}
impl<'a> ClassifiedAttrs<'a> {
fn effective_db_system(&self) -> Option<&'a str> {
self.db_system_name
.filter(|s| !s.trim().is_empty())
.or_else(|| self.db_system.filter(|s| !s.trim().is_empty()))
.or_else(|| self.db_type.filter(|s| !s.trim().is_empty()))
}
fn code_attrs(&self) -> CodeAttrs<'a> {
let function_name = self.code_function_name.or(self.code_function);
let filepath = self.code_file_path.or(self.code_filepath);
let lineno = self.code_line_number.or(self.code_lineno);
let namespace = self.code_namespace.or_else(|| {
self.code_function_name
.and_then(super::namespace_from_qualified_name)
});
CodeAttrs {
function_name,
filepath,
lineno,
namespace,
}
}
}
fn classify_span_attrs(attrs: &[KeyValue]) -> ClassifiedAttrs<'_> {
let mut out = ClassifiedAttrs::default();
for kv in attrs {
let value = kv.value.as_ref().and_then(|v| v.value.as_ref());
match kv.key.as_str() {
"db.statement" => out.db_statement = any_value_as_str(value),
"db.query.text" => out.db_query_text = any_value_as_str(value),
"db.system" => out.db_system = any_value_as_str(value),
"db.system.name" => out.db_system_name = any_value_as_str(value),
"dd.span.Resource" => out.dd_resource = any_value_as_str(value),
"db.type" => out.db_type = any_value_as_str(value),
"http.url" => out.http_url = any_value_as_str(value),
"url.full" => out.url_full = any_value_as_str(value),
"http.method" => out.http_method = any_value_as_str(value),
"http.request.method" => out.http_request_method = any_value_as_str(value),
"rpc.system" => out.rpc_system = any_value_as_str(value),
"rpc.service" => out.rpc_service = any_value_as_str(value),
"rpc.method" => out.rpc_method = any_value_as_str(value),
"messaging.system" => out.messaging_system = any_value_as_str(value),
"messaging.destination.name" => {
out.messaging_destination_name = any_value_as_str(value);
}
"messaging.destination" => out.messaging_destination = any_value_as_str(value),
"messaging.message.body.size" => out.messaging_body_size = any_value_as_int(value),
"http.status_code" => out.http_status_code = any_value_as_int(value),
"http.response.status_code" => out.http_response_status_code = any_value_as_int(value),
"http.response.body.size" => out.http_response_body_size = any_value_as_int(value),
"http.response_content_length" => {
out.http_response_content_length = any_value_as_int(value);
}
"cloud.region" => out.cloud_region = any_value_as_str(value),
"code.function.name" => out.code_function_name = any_value_as_str(value),
"code.function" => out.code_function = any_value_as_str(value),
"code.file.path" => out.code_file_path = any_value_as_str(value),
"code.filepath" => out.code_filepath = any_value_as_str(value),
"code.line.number" => out.code_line_number = any_value_as_int(value),
"code.lineno" => out.code_lineno = any_value_as_int(value),
"code.namespace" => out.code_namespace = any_value_as_str(value),
_ => {}
}
}
out
}
fn read_code_attrs(attrs: &[KeyValue]) -> CodeAttrs<'_> {
let mut function_name_stable = None;
let mut function_name_legacy = None;
let mut filepath_stable = None;
let mut filepath_legacy = None;
let mut lineno_stable = None;
let mut lineno_legacy = None;
let mut namespace_explicit = None;
for kv in attrs {
let value = kv.value.as_ref().and_then(|v| v.value.as_ref());
match kv.key.as_str() {
"code.function.name" => function_name_stable = any_value_as_str(value),
"code.function" => function_name_legacy = any_value_as_str(value),
"code.file.path" => filepath_stable = any_value_as_str(value),
"code.filepath" => filepath_legacy = any_value_as_str(value),
"code.line.number" => lineno_stable = any_value_as_int(value),
"code.lineno" => lineno_legacy = any_value_as_int(value),
"code.namespace" => namespace_explicit = any_value_as_str(value),
_ => {}
}
}
let namespace = namespace_explicit
.or_else(|| function_name_stable.and_then(super::namespace_from_qualified_name));
CodeAttrs {
function_name: function_name_stable.or(function_name_legacy),
filepath: filepath_stable.or(filepath_legacy),
lineno: lineno_stable.or(lineno_legacy),
namespace,
}
}
fn walk_parents_for_code_attrs<'a>(
leaf: CodeAttrs<'a>,
parent_span_id: &[u8],
span_index: &HashMap<&[u8], &'a Span>,
) -> CodeAttrs<'a> {
if leaf.has_any() || parent_span_id.is_empty() {
return leaf;
}
let mut current_parent_id = parent_span_id;
let mut depth = 0;
loop {
let Some(parent) = span_index.get(current_parent_id) else {
return CodeAttrs::default();
};
let attrs = read_code_attrs(&parent.attributes);
if attrs.has_any() {
return attrs;
}
if parent.parent_span_id.is_empty() || depth >= CODE_ATTRS_MAX_DEPTH {
return CodeAttrs::default();
}
current_parent_id = parent.parent_span_id.as_slice();
depth += 1;
}
}
fn resolve_producer_link<'a>(
span: &'a Span,
span_index: &HashMap<&'a [u8], &'a Span>,
consumers_by_parent: &HashMap<&'a [u8], Vec<&'a Span>>,
) -> Option<Arc<str>> {
let valid =
|id: &[u8]| id.len() == TRACE_ID_LEN && id != span.trace_id && id.iter().any(|&b| b != 0);
let link_of = |s: &Span| {
s.links
.first()
.filter(|l| valid(&l.trace_id))
.map(|l| Arc::from(bytes_to_hex(&l.trace_id).as_str()))
};
let sibling_link = |parent_id: &[u8], started: u64| {
if parent_id.is_empty() || parent_id.iter().all(|&b| b == 0) {
return None;
}
consumers_by_parent
.get(parent_id)?
.iter()
.filter(|c| {
c.trace_id == span.trace_id
&& c.span_id != span.span_id
&& started >= c.start_time_unix_nano
})
.max_by_key(|c| (c.start_time_unix_nano, &c.span_id))
.and_then(|c| link_of(c))
};
if let Some(found) = sibling_link(&span.parent_span_id, span.start_time_unix_nano) {
return Some(found);
}
let mut found = None;
walk_same_trace_ancestors(span, span_index, |ancestor| {
if ancestor.kind == opentelemetry_proto::tonic::trace::v1::span::SpanKind::Consumer as i32 {
found = link_of(ancestor);
}
if found.is_none() {
found = sibling_link(&ancestor.parent_span_id, ancestor.start_time_unix_nano);
}
found.is_some()
});
found
}
fn build_consumer_link_indexes(request: &ExportTraceServiceRequest) -> ServiceConsumerIndexes<'_> {
let mut per_service: ServiceConsumerIndexes<'_> = HashMap::new();
let mut kept_per_service: HashMap<&str, usize> = HashMap::new();
for resource_spans in &request.resource_spans {
let service = resource_service_name(resource_spans);
index_linked_consumers(
resource_spans,
per_service.entry(service).or_default(),
kept_per_service.entry(service).or_default(),
);
}
per_service
}
fn any_linked_consumer(request: &ExportTraceServiceRequest) -> bool {
request.resource_spans.iter().any(|resource_spans| {
resource_spans.scope_spans.iter().any(|scope_spans| {
scope_spans.spans.iter().any(|span| {
span.kind == opentelemetry_proto::tonic::trace::v1::span::SpanKind::Consumer as i32
&& !span.links.is_empty()
})
})
})
}
fn index_linked_consumers<'a>(
resource_spans: &'a opentelemetry_proto::tonic::trace::v1::ResourceSpans,
index: &mut HashMap<&'a [u8], Vec<&'a Span>>,
kept: &mut usize,
) {
let linked_consumers = resource_spans
.scope_spans
.iter()
.flat_map(|scope_spans| &scope_spans.spans)
.filter(|span| {
!span.links.is_empty()
&& span.kind
== opentelemetry_proto::tonic::trace::v1::span::SpanKind::Consumer as i32
});
for span in linked_consumers {
if *kept >= MAX_SPANS_PER_SERVICE {
if *kept == MAX_SPANS_PER_SERVICE {
tracing::warn!(
"OTLP consumer-link index capped at {} entries for one service, producer links may be missing for its remaining spans",
MAX_SPANS_PER_SERVICE
);
*kept += 1;
}
return;
}
index.entry(&span.parent_span_id).or_default().push(span);
*kept += 1;
}
}
fn inbound_http_endpoint(span: &Span) -> Option<&str> {
let usable = |s: &&str| !s.trim().is_empty();
get_str_attribute(&span.attributes, "http.route")
.filter(usable)
.or_else(|| {
if span.kind == opentelemetry_proto::tonic::trace::v1::span::SpanKind::Client as i32 {
return None;
}
get_str_attribute(&span.attributes, "http.url")
.or_else(|| get_str_attribute(&span.attributes, "url.full"))
.filter(usable)
})
}
fn resolve_source_endpoint<'a>(
leaf: CodeAttrs<'a>,
parent_span_id: &[u8],
span_index: &HashMap<&[u8], &'a Span>,
) -> String {
let mut outermost_frame =
crate::ingest::code_frame_endpoint(leaf.namespace, leaf.function_name);
let mut current_parent_id = parent_span_id;
let mut depth = 0;
while !current_parent_id.is_empty() {
let Some(parent) = span_index.get(current_parent_id) else {
break;
};
if let Some(route) = inbound_http_endpoint(parent) {
return route.to_string();
}
let attrs = read_code_attrs(&parent.attributes);
if let Some(frame) =
crate::ingest::code_frame_endpoint(attrs.namespace, attrs.function_name)
{
outermost_frame = Some(frame);
}
if depth >= CODE_ATTRS_MAX_DEPTH {
break;
}
current_parent_id = parent.parent_span_id.as_slice();
depth += 1;
}
outermost_frame.unwrap_or_else(|| "unknown".to_string())
}
fn build_span_indexes(request: &ExportTraceServiceRequest) -> ServiceSpanIndexes<'_> {
let mut per_service: ServiceSpanIndexes<'_> = HashMap::new();
for resource_spans in &request.resource_spans {
let index = per_service
.entry(resource_service_name(resource_spans))
.or_default();
for scope_spans in &resource_spans.scope_spans {
for span in &scope_spans.spans {
if span.span_id.is_empty() {
continue;
}
if index.len() >= MAX_SPANS_PER_SERVICE {
tracing::warn!(
"OTLP span index capped at {} entries for one service, parent lookup may be degraded for its remaining spans",
MAX_SPANS_PER_SERVICE
);
break;
}
index.insert(&span.span_id, span);
}
}
}
per_service
}
fn resource_service_name(
resource_spans: &opentelemetry_proto::tonic::trace::v1::ResourceSpans,
) -> &str {
resource_spans
.resource
.as_ref()
.and_then(|r| get_str_attribute(&r.attributes, "service.name"))
.unwrap_or("unknown")
}
fn build_scope_index(
resource_spans: &opentelemetry_proto::tonic::trace::v1::ResourceSpans,
) -> HashMap<&[u8], &str> {
let mut index: HashMap<&[u8], &str> = HashMap::new();
let mut count = 0usize;
'outer: for scope_spans in &resource_spans.scope_spans {
let scope_name = scope_spans.scope.as_ref().map_or("", |s| s.name.as_str());
if scope_name.is_empty() {
continue;
}
for span in &scope_spans.spans {
index.insert(&span.span_id, scope_name);
count += 1;
if count >= MAX_SPANS_PER_SERVICE {
break 'outer;
}
}
}
index
}
fn collect_instrumentation_scopes(
span: &Span,
span_index: &HashMap<&[u8], &Span>,
scope_index: &HashMap<&[u8], &str>,
) -> Vec<Arc<str>> {
let mut out: Vec<Arc<str>> = Vec::new();
let mut current = span;
let mut depth = 0;
loop {
if let Some(name) = scope_index.get(current.span_id.as_slice())
&& !out.iter().any(|s| s.as_ref() == *name)
{
out.push(Arc::from(*name));
}
if current.parent_span_id.is_empty() || depth >= CODE_ATTRS_MAX_DEPTH {
return out;
}
let Some(parent) = span_index.get(current.parent_span_id.as_slice()) else {
return out;
};
current = *parent;
depth += 1;
}
}
fn has_http_signal(c: &ClassifiedAttrs<'_>) -> bool {
c.http_url.is_some()
|| c.url_full.is_some()
|| c.http_method.is_some()
|| c.http_request_method.is_some()
}
fn resolve_sql_statement<'a>(c: &ClassifiedAttrs<'a>, db_system: Option<&str>) -> Option<&'a str> {
c.db_statement.or(c.db_query_text).or_else(|| {
c.dd_resource
.map(str::trim)
.filter(|s| !s.is_empty())
.filter(|_| {
!has_http_signal(c) && db_system.is_some_and(crate::ingest::is_sql_db_system)
})
})
}
type SpanKey<'a> = (&'a [u8], &'a [u8]);
fn span_key(span: &Span) -> SpanKey<'_> {
(span.trace_id.as_slice(), span.span_id.as_slice())
}
enum StitchDecision<'a> {
Suppress,
Adopt(&'a str),
}
struct StitchDonor<'a> {
span: &'a Span,
statement: &'a str,
}
const SIBLING_DONOR_LOOKBACK: usize = 8;
fn walk_same_trace_ancestors<'a>(
span: &'a Span,
span_index: &HashMap<&'a [u8], &'a Span>,
mut visit: impl FnMut(&'a Span) -> bool,
) {
let mut current = span;
for _ in 0..CODE_ATTRS_MAX_DEPTH {
if current.parent_span_id.is_empty() {
return;
}
let Some(&parent) = span_index.get(current.parent_span_id.as_slice()) else {
return;
};
if parent.trace_id != span.trace_id || parent.span_id == span.span_id {
return;
}
if visit(parent) {
return;
}
current = parent;
}
}
fn looks_like_query_execution(name: &str) -> bool {
let name = name.to_ascii_lowercase();
name.contains("execute") || name.contains("query")
}
fn classify_resource_spans(
resource_spans: &opentelemetry_proto::tonic::trace::v1::ResourceSpans,
) -> Vec<ClassifiedAttrs<'_>> {
let total: usize = resource_spans
.scope_spans
.iter()
.map(|s| s.spans.len())
.sum();
let mut out = Vec::with_capacity(total.min(MAX_SPANS_PER_SERVICE));
'outer: for scope_spans in &resource_spans.scope_spans {
for span in &scope_spans.spans {
if out.len() >= MAX_SPANS_PER_SERVICE {
break 'outer;
}
out.push(classify_span_attrs(&span.attributes));
}
}
out
}
enum SpanRole<'a> {
Donor(&'a str),
Orphan { has_sql_db_system: bool },
Skip,
}
fn classify_stitch_role<'a>(span: &Span, c: &ClassifiedAttrs<'a>) -> SpanRole<'a> {
if span.trace_id.is_empty() || span.span_id.is_empty() {
return SpanRole::Skip;
}
let db_system = c
.effective_db_system()
.map(crate::ingest::canonical_db_system);
if db_system.is_some_and(crate::ingest::is_non_sql_db_system) {
return SpanRole::Skip;
}
if let Some(statement) = resolve_sql_statement(c, db_system) {
SpanRole::Donor(statement)
} else if !has_http_signal(c)
&& c.rpc_system.is_none()
&& c.messaging_system.is_none()
&& looks_like_query_execution(&span.name)
{
SpanRole::Orphan {
has_sql_db_system: db_system.is_some_and(crate::ingest::is_sql_db_system),
}
} else {
SpanRole::Skip
}
}
fn collect_stitch_participants<'a>(
resource_spans: &'a opentelemetry_proto::tonic::trace::v1::ResourceSpans,
classified: &[ClassifiedAttrs<'a>],
) -> (Vec<StitchDonor<'a>>, Vec<&'a Span>) {
let mut donors = Vec::new();
let mut provisional: Vec<(&'a Span, bool)> = Vec::new();
let mut idx = 0usize;
'outer: for scope_spans in &resource_spans.scope_spans {
for span in &scope_spans.spans {
let Some(c) = classified.get(idx) else {
break 'outer;
};
idx += 1;
match classify_stitch_role(span, c) {
SpanRole::Donor(statement) => donors.push(StitchDonor { span, statement }),
SpanRole::Orphan { has_sql_db_system } => {
provisional.push((span, has_sql_db_system));
}
SpanRole::Skip => {}
}
}
}
let donor_parents: HashSet<SpanKey<'a>> = donors
.iter()
.filter(|d| !d.span.parent_span_id.is_empty())
.map(|d| (d.span.trace_id.as_slice(), d.span.parent_span_id.as_slice()))
.collect();
let orphans = provisional
.into_iter()
.filter(|&(span, is_sql)| {
is_sql
|| donor_parents
.contains(&(span.trace_id.as_slice(), span.parent_span_id.as_slice()))
})
.map(|(span, _)| span)
.collect();
(donors, orphans)
}
fn suppress_layered_duplicates<'a>(
donors: &[StitchDonor<'a>],
donor_by_id: &HashMap<SpanKey<'a>, usize>,
span_index: &HashMap<&'a [u8], &'a Span>,
) -> Vec<bool> {
let mut suppressed = vec![false; donors.len()];
for (i, donor) in donors.iter().enumerate() {
let mut suppressor = None;
walk_same_trace_ancestors(donor.span, span_index, |ancestor| {
suppressor = donor_by_id
.get(&span_key(ancestor))
.copied()
.filter(|&j| donors[j].statement == donor.statement);
suppressor.is_some()
});
if let Some(j) = suppressor {
let mut mutual = false;
walk_same_trace_ancestors(donors[j].span, span_index, |ancestor| {
mutual = ancestor.span_id == donor.span.span_id;
mutual
});
suppressed[i] = !mutual;
}
}
suppressed
}
fn split_layered_orphans<'a>(
orphans: &[&'a Span],
span_index: &HashMap<&'a [u8], &'a Span>,
) -> (Vec<(&'a Span, SpanKey<'a>)>, Vec<&'a Span>) {
let orphan_keys: HashSet<SpanKey<'a>> = orphans.iter().map(|o| span_key(o)).collect();
let mut deferred = Vec::new();
let mut carriers = Vec::new();
for &orphan in orphans {
let mut carrier_key = None;
walk_same_trace_ancestors(orphan, span_index, |ancestor| {
if orphan_keys.contains(&span_key(ancestor)) {
carrier_key = Some(span_key(ancestor));
}
false
});
match carrier_key {
Some(key) => deferred.push((orphan, key)),
None => carriers.push(orphan),
}
}
(deferred, carriers)
}
fn push_sibling_candidates(
donors: &[StitchDonor<'_>],
consumed: &[bool],
siblings: &[usize],
orphan_start: u64,
candidates: &mut Vec<usize>,
) {
let at_or_before =
siblings.partition_point(|&i| donors[i].span.start_time_unix_nano <= orphan_start);
if at_or_before == 0 {
return;
}
let best_start = donors[siblings[at_or_before - 1]].span.start_time_unix_nano;
let run_start = siblings[..at_or_before]
.partition_point(|&i| donors[i].span.start_time_unix_nano < best_start);
candidates.push(siblings[run_start]);
for &i in siblings[..at_or_before]
.iter()
.rev()
.take(SIBLING_DONOR_LOOKBACK)
{
if !consumed[i] {
candidates.push(i);
return;
}
}
}
fn nearest_donor(
donors: &[StitchDonor<'_>],
consumed: &[bool],
candidates: &[usize],
orphan_start: u64,
) -> Option<usize> {
let mut best: Option<(usize, u64, bool)> = None;
for &i in candidates {
let start = donors[i].span.start_time_unix_nano;
if start > orphan_start {
continue;
}
let free = !consumed[i];
let better = match best {
None => true,
Some((_, b_start, b_free)) => (free && !b_free) || (free == b_free && start > b_start),
};
if better {
best = Some((i, start, free));
}
}
best.map(|(i, _, _)| i).or_else(|| {
candidates
.iter()
.copied()
.min_by_key(|&i| donors[i].span.start_time_unix_nano.abs_diff(orphan_start))
})
}
fn bucket_surviving_donors<'a>(
donors: &[StitchDonor<'a>],
donor_suppressed: &[bool],
span_index: &HashMap<&'a [u8], &'a Span>,
) -> (
HashMap<SpanKey<'a>, Vec<usize>>,
HashMap<SpanKey<'a>, Vec<usize>>,
) {
let mut donors_by_parent: HashMap<SpanKey<'a>, Vec<usize>> = HashMap::new();
let mut donors_by_ancestor: HashMap<SpanKey<'a>, Vec<usize>> = HashMap::new();
for (i, donor) in donors.iter().enumerate() {
if donor_suppressed[i] {
continue;
}
if !donor.span.parent_span_id.is_empty() {
donors_by_parent
.entry((
donor.span.trace_id.as_slice(),
donor.span.parent_span_id.as_slice(),
))
.or_default()
.push(i);
}
walk_same_trace_ancestors(donor.span, span_index, |ancestor| {
donors_by_ancestor
.entry(span_key(ancestor))
.or_default()
.push(i);
false
});
}
for bucket in donors_by_parent.values_mut() {
bucket.sort_unstable_by_key(|&i| (donors[i].span.start_time_unix_nano, i));
}
(donors_by_parent, donors_by_ancestor)
}
#[allow(clippy::too_many_arguments)]
fn collect_orphan_candidates<'a>(
orphan: &'a Span,
donors: &[StitchDonor<'a>],
donor_by_id: &HashMap<SpanKey<'a>, usize>,
donor_suppressed: &[bool],
donor_consumed: &[bool],
donors_by_parent: &HashMap<SpanKey<'a>, Vec<usize>>,
donors_by_ancestor: &HashMap<SpanKey<'a>, Vec<usize>>,
span_index: &HashMap<&'a [u8], &'a Span>,
candidates: &mut Vec<usize>,
) {
if !orphan.parent_span_id.is_empty()
&& let Some(siblings) =
donors_by_parent.get(&(orphan.trace_id.as_slice(), orphan.parent_span_id.as_slice()))
{
push_sibling_candidates(
donors,
donor_consumed,
siblings,
orphan.start_time_unix_nano,
candidates,
);
}
walk_same_trace_ancestors(orphan, span_index, |ancestor| {
if let Some(&i) = donor_by_id.get(&span_key(ancestor))
&& !donor_suppressed[i]
{
candidates.push(i);
}
false
});
if let Some(descendants) = donors_by_ancestor.get(&span_key(orphan)) {
candidates.extend(descendants.iter().copied());
}
}
fn compute_stitch_decisions<'a>(
resource_spans: &'a opentelemetry_proto::tonic::trace::v1::ResourceSpans,
span_index: &HashMap<&'a [u8], &'a Span>,
classified: &[ClassifiedAttrs<'a>],
) -> HashMap<SpanKey<'a>, StitchDecision<'a>> {
let (donors, orphans) = collect_stitch_participants(resource_spans, classified);
if donors.is_empty() {
return HashMap::new();
}
let donor_by_id: HashMap<SpanKey<'a>, usize> = donors
.iter()
.enumerate()
.map(|(i, d)| (span_key(d.span), i))
.collect();
let donor_suppressed = suppress_layered_duplicates(&donors, &donor_by_id, span_index);
let (deferred, carriers) = split_layered_orphans(&orphans, span_index);
let mut decisions: HashMap<SpanKey<'a>, StitchDecision<'a>> = HashMap::new();
let mut donor_consumed = vec![false; donors.len()];
let mut stitched: HashSet<SpanKey<'a>> = HashSet::new();
if !carriers.is_empty() {
let (donors_by_parent, donors_by_ancestor) =
bucket_surviving_donors(&donors, &donor_suppressed, span_index);
let mut candidates: Vec<usize> = Vec::new();
for orphan in carriers {
candidates.clear();
collect_orphan_candidates(
orphan,
&donors,
&donor_by_id,
&donor_suppressed,
&donor_consumed,
&donors_by_parent,
&donors_by_ancestor,
span_index,
&mut candidates,
);
if let Some(i) = nearest_donor(
&donors,
&donor_consumed,
&candidates,
orphan.start_time_unix_nano,
) {
decisions.insert(span_key(orphan), StitchDecision::Adopt(donors[i].statement));
donor_consumed[i] = true;
stitched.insert(span_key(orphan));
}
}
}
for (i, donor) in donors.iter().enumerate() {
if donor_suppressed[i] || donor_consumed[i] {
decisions.insert(span_key(donor.span), StitchDecision::Suppress);
}
}
for (span, carrier_key) in deferred {
if stitched.contains(&carrier_key) {
decisions.insert(span_key(span), StitchDecision::Suppress);
}
}
decisions
}
#[must_use]
pub fn convert_otlp_request(request: &ExportTraceServiceRequest) -> Vec<SpanEvent> {
convert_otlp_request_counted(request).0
}
#[must_use]
pub fn convert_otlp_request_counted(
request: &ExportTraceServiceRequest,
) -> (Vec<SpanEvent>, SpanConversionStats) {
let mut events = Vec::new();
let mut stats = SpanConversionStats::default();
let span_indexes = build_span_indexes(request);
let empty_index = HashMap::new();
let empty_consumers = HashMap::new();
let consumer_indexes = if any_linked_consumer(request) {
build_consumer_link_indexes(request)
} else {
HashMap::new()
};
for resource_spans in &request.resource_spans {
let service_name = resource_service_name(resource_spans);
convert_resource_spans(
resource_spans,
span_indexes.get(service_name).unwrap_or(&empty_index),
consumer_indexes
.get(service_name)
.unwrap_or(&empty_consumers),
&mut events,
&mut stats,
);
}
(events, stats)
}
fn convert_resource_spans<'a>(
resource_spans: &'a opentelemetry_proto::tonic::trace::v1::ResourceSpans,
span_index: &HashMap<&'a [u8], &'a Span>,
consumer_index: &HashMap<&'a [u8], Vec<&'a Span>>,
events: &mut Vec<SpanEvent>,
stats: &mut SpanConversionStats,
) {
let service_arc: Arc<str> = Arc::from(resource_service_name(resource_spans));
let resource_cloud_region: Option<Arc<str>> = resource_spans
.resource
.as_ref()
.and_then(|r| get_str_attribute(&r.attributes, "cloud.region"))
.filter(|s| crate::score::carbon::is_valid_region_id(s))
.map(Arc::from);
let scope_index = build_scope_index(resource_spans);
let classified = classify_resource_spans(resource_spans);
let stitch = compute_stitch_decisions(resource_spans, span_index, &classified);
let mut span_idx = 0usize;
for scope_spans in &resource_spans.scope_spans {
for span in &scope_spans.spans {
stats.received += 1;
let cached_attrs = classified.get(span_idx);
span_idx += 1;
let stitched_statement = match stitch.get(&span_key(span)) {
Some(StitchDecision::Suppress) => {
stats.count_filtered(OtlpSpanFilterReason::MergedDbSpan);
continue;
}
Some(StitchDecision::Adopt(statement)) => Some(*statement),
None => None,
};
match convert_span(
span,
&service_arc,
resource_cloud_region.as_ref(),
span_index,
&scope_index,
stitched_statement,
cached_attrs,
consumer_index,
) {
Ok(event) => events.push(event),
Err(reason) => stats.count_filtered(reason),
}
}
}
}
fn span_filter_reason(
classified: &ClassifiedAttrs<'_>,
db_system: Option<&str>,
kind: i32,
) -> OtlpSpanFilterReason {
let server = kind == opentelemetry_proto::tonic::trace::v1::span::SpanKind::Server as i32;
if db_system.is_some() {
OtlpSpanFilterReason::MissingDbStatement
} else if !server
&& classified
.http_method
.or(classified.http_request_method)
.is_some()
{
OtlpSpanFilterReason::MissingHttpUrl
} else {
OtlpSpanFilterReason::NotIo
}
}
fn classify_io_event(
c: &ClassifiedAttrs<'_>,
db_system: Option<&str>,
span_name: &str,
kind: i32,
) -> Option<(EventType, String, String)> {
if let Some(statement) = resolve_sql_statement(c, db_system) {
let op = db_system.unwrap_or("sql").to_string();
Some((EventType::Sql, statement.to_string(), op))
} else if let Some(url) = c.http_url.or(c.url_full) {
let method = c
.http_method
.or(c.http_request_method)
.unwrap_or("GET")
.to_string();
Some((EventType::HttpOut, url.to_string(), method))
} else if let Some(system) = c.rpc_system.filter(|_| {
kind == opentelemetry_proto::tonic::trace::v1::span::SpanKind::Client as i32
}) {
let svc = c.rpc_service.filter(|s| !s.is_empty());
let method = c.rpc_method.filter(|s| !s.is_empty());
let target = match (svc, method) {
(Some(svc), Some(method)) => format!("{svc}/{method}"),
_ => span_name.to_string(),
};
if target.is_empty() {
return None;
}
Some((EventType::HttpOut, target, system.to_string()))
} else if c.messaging_system.is_some() {
classify_messaging_event(c, span_name, kind)
} else {
None
}
}
fn payload_size_bytes(event_type: &EventType, c: &ClassifiedAttrs<'_>) -> Option<u64> {
match event_type {
EventType::HttpOut => c.http_response_body_size.or(c.http_response_content_length),
EventType::Messaging => c.messaging_body_size,
EventType::Sql => None,
}
.and_then(|v| u64::try_from(v).ok())
}
fn classify_messaging_event(
c: &ClassifiedAttrs<'_>,
span_name: &str,
kind: i32,
) -> Option<(EventType, String, String)> {
let system = c.messaging_system.filter(|s| !s.trim().is_empty())?;
if kind != opentelemetry_proto::tonic::trace::v1::span::SpanKind::Producer as i32 {
return None;
}
let target = c
.messaging_destination_name
.map(str::trim)
.filter(|s| !s.is_empty())
.or_else(|| {
c.messaging_destination
.map(str::trim)
.filter(|s| !s.is_empty())
})
.map_or_else(|| span_name.trim().to_string(), ToString::to_string);
if target.is_empty() {
return None;
}
Some((EventType::Messaging, target, system.to_string()))
}
fn rebuilt_classified<'a>(
span: &'a Span,
cached_attrs: Option<&ClassifiedAttrs<'a>>,
stitched_statement: Option<&'a str>,
) -> Option<ClassifiedAttrs<'a>> {
if cached_attrs.is_some() && stitched_statement.is_none() {
return None;
}
let mut rebuilt = classify_span_attrs(&span.attributes);
if stitched_statement.is_some() {
rebuilt.db_statement = stitched_statement;
}
Some(rebuilt)
}
#[allow(clippy::too_many_arguments)] fn convert_span<'a>(
span: &'a Span,
service_arc: &Arc<str>,
resource_cloud_region: Option<&Arc<str>>,
span_index: &HashMap<&[u8], &Span>,
scope_index: &HashMap<&[u8], &str>,
stitched_statement: Option<&'a str>,
cached_attrs: Option<&ClassifiedAttrs<'a>>,
consumer_index: &HashMap<&'a [u8], Vec<&'a Span>>,
) -> Result<SpanEvent, OtlpSpanFilterReason> {
let owned = rebuilt_classified(span, cached_attrs, stitched_statement);
let classified = match (&owned, cached_attrs) {
(Some(rebuilt), _) => rebuilt,
(None, Some(cached)) => cached,
(None, None) => unreachable!("rebuilt_classified rebuilds on cache miss"),
};
let db_system = classified
.effective_db_system()
.map(crate::ingest::canonical_db_system);
if db_system.is_some_and(crate::ingest::is_non_sql_db_system) {
return Err(OtlpSpanFilterReason::NonSqlDatastore);
}
let Some((event_type, target, operation)) =
classify_io_event(classified, db_system, &span.name, span.kind)
else {
return Err(span_filter_reason(classified, db_system, span.kind));
};
let start_nanos = span.start_time_unix_nano;
let end_nanos = span.end_time_unix_nano;
let timestamp = nanos_to_iso8601(start_nanos);
if end_nanos < start_nanos {
tracing::trace!("Span has end_time < start_time (clock skew?), duration forced to 0");
}
let duration_us = end_nanos.saturating_sub(start_nanos) / 1000;
let trace_id = bytes_to_hex(&span.trace_id);
let span_id = bytes_to_hex(&span.span_id);
let status_code = if event_type == EventType::HttpOut {
classified
.http_status_code
.or(classified.http_response_status_code)
.and_then(|c| u16::try_from(c).ok())
} else {
None
};
let response_size_bytes = payload_size_bytes(&event_type, classified);
let code =
walk_parents_for_code_attrs(classified.code_attrs(), &span.parent_span_id, span_index);
let source_method = if span.parent_span_id.is_empty() {
span.name.clone()
} else if let Some(parent) = span_index.get(span.parent_span_id.as_slice()) {
get_str_attribute(&parent.attributes, "code.function")
.map_or_else(|| parent.name.clone(), ToString::to_string)
} else {
span.name.clone()
};
let source_endpoint =
resolve_source_endpoint(classified.code_attrs(), &span.parent_span_id, span_index);
let parent_span_id = if span.parent_span_id.is_empty() {
None
} else {
Some(bytes_to_hex(&span.parent_span_id))
};
let cloud_region: Option<Arc<str>> = resource_cloud_region.cloned().or_else(|| {
classified
.cloud_region
.filter(|s| crate::score::carbon::is_valid_region_id(s))
.map(Arc::from)
});
let code_function: Option<Arc<str>> = code.function_name.map(Arc::from);
let code_filepath: Option<Arc<str>> = code.filepath.map(Arc::from);
let code_lineno = code.lineno.and_then(|v| u32::try_from(v).ok());
let code_namespace: Option<Arc<str>> = code.namespace.map(Arc::from);
let instrumentation_scopes = collect_instrumentation_scopes(span, span_index, scope_index);
let link_trace_id = (!consumer_index.is_empty())
.then(|| resolve_producer_link(span, span_index, consumer_index))
.flatten();
let mut event = SpanEvent {
timestamp,
trace_id,
span_id,
parent_span_id,
link_trace_id,
service: Arc::clone(service_arc),
cloud_region,
event_type,
operation,
target,
duration_us,
source: EventSource {
endpoint: source_endpoint,
method: source_method,
},
status_code,
response_size_bytes,
code_function,
code_filepath,
code_lineno,
code_namespace,
instrumentation_scopes,
};
crate::event::sanitize_span_event(&mut event);
Ok(event)
}
const INGEST_ENQUEUE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(2);
#[derive(Clone)]
pub enum OtlpSink {
Events(tokio::sync::mpsc::Sender<Vec<SpanEvent>>),
Raw(tokio::sync::mpsc::Sender<ExportTraceServiceRequest>),
}
pub(crate) enum SinkRejection {
Full,
Closed,
}
impl<T> From<tokio::sync::mpsc::error::SendTimeoutError<T>> for SinkRejection {
fn from(e: tokio::sync::mpsc::error::SendTimeoutError<T>) -> Self {
match e {
tokio::sync::mpsc::error::SendTimeoutError::Timeout(_) => Self::Full,
tokio::sync::mpsc::error::SendTimeoutError::Closed(_) => Self::Closed,
}
}
}
impl OtlpSink {
async fn accept(
&self,
request: ExportTraceServiceRequest,
metrics: Option<&Arc<dyn MetricsSink>>,
) -> Result<(), SinkRejection> {
match self {
Self::Events(tx) => {
let (events, stats) = convert_otlp_request_counted(&request);
if let Some(m) = metrics {
m.record_otlp_spans(stats);
}
if events.is_empty() {
return Ok(());
}
Ok(tx.send_timeout(events, INGEST_ENQUEUE_TIMEOUT).await?)
}
Self::Raw(tx) => Ok(tx.send_timeout(request, INGEST_ENQUEUE_TIMEOUT).await?),
}
}
}
pub struct OtlpGrpcService {
sink: OtlpSink,
metrics: Option<Arc<dyn MetricsSink>>,
}
impl OtlpGrpcService {
#[must_use]
pub fn new(
sender: tokio::sync::mpsc::Sender<Vec<SpanEvent>>,
metrics: Option<Arc<dyn MetricsSink>>,
) -> Self {
Self {
sink: OtlpSink::Events(sender),
metrics,
}
}
#[must_use]
pub fn new_raw(
sender: tokio::sync::mpsc::Sender<ExportTraceServiceRequest>,
metrics: Option<Arc<dyn MetricsSink>>,
) -> Self {
Self {
sink: OtlpSink::Raw(sender),
metrics,
}
}
}
#[async_trait]
impl opentelemetry_proto::tonic::collector::trace::v1::trace_service_server::TraceService
for OtlpGrpcService
{
async fn export(
&self,
request: Request<ExportTraceServiceRequest>,
) -> Result<Response<ExportTraceServiceResponse>, Status> {
if let Some(m) = self.metrics.as_ref()
&& m.ingest_over_memory_limit()
{
m.record_otlp_reject(OtlpRejectReason::MemoryPressure);
return Err(Status::unavailable(
"ingest paused: memory high-water, retry",
));
}
if let Err(e) = self
.sink
.accept(request.into_inner(), self.metrics.as_ref())
.await
{
if let Some(m) = self.metrics.as_ref() {
m.record_otlp_reject(OtlpRejectReason::ChannelFull);
}
return Err(match e {
SinkRejection::Full => Status::unavailable("ingest queue full, retry"),
SinkRejection::Closed => Status::internal("event channel closed"),
});
}
Ok(Response::new(ExportTraceServiceResponse {
partial_success: None,
}))
}
}
#[derive(Clone)]
struct OtlpHttpState {
sink: OtlpSink,
metrics: Option<Arc<dyn MetricsSink>>,
}
pub fn otlp_http_router(
sender: tokio::sync::mpsc::Sender<Vec<SpanEvent>>,
max_payload_size: usize,
metrics: Option<Arc<dyn MetricsSink>>,
) -> axum::Router {
otlp_http_router_with_sink(OtlpSink::Events(sender), max_payload_size, metrics)
}
pub fn otlp_http_router_with_sink(
sink: OtlpSink,
max_payload_size: usize,
metrics: Option<Arc<dyn MetricsSink>>,
) -> axum::Router {
use axum::{
Router,
extract::State,
http::{HeaderMap, StatusCode, header},
routing::post,
};
fn is_protobuf_content_type(headers: &HeaderMap) -> bool {
headers
.get(header::CONTENT_TYPE)
.and_then(|v| v.to_str().ok())
.is_some_and(|ct| {
ct.split(';')
.next()
.unwrap_or("")
.trim()
.eq_ignore_ascii_case("application/x-protobuf")
})
}
async fn handle_traces(
State(state): State<OtlpHttpState>,
headers: HeaderMap,
body: axum::body::Bytes,
) -> StatusCode {
if let Some(m) = state.metrics.as_ref()
&& m.ingest_over_memory_limit()
{
m.record_otlp_reject(OtlpRejectReason::MemoryPressure);
return StatusCode::SERVICE_UNAVAILABLE;
}
let reject = |reason: OtlpRejectReason| {
if let Some(m) = state.metrics.as_ref() {
m.record_otlp_reject(reason);
}
};
if !is_protobuf_content_type(&headers) {
reject(OtlpRejectReason::UnsupportedMediaType);
return StatusCode::UNSUPPORTED_MEDIA_TYPE;
}
let Ok(request) = <ExportTraceServiceRequest as prost::Message>::decode(body.as_ref())
else {
reject(OtlpRejectReason::ParseError);
return StatusCode::BAD_REQUEST;
};
if state
.sink
.accept(request, state.metrics.as_ref())
.await
.is_err()
{
tracing::warn!("OTLP HTTP: ingest channel full or closed, dropping request");
reject(OtlpRejectReason::ChannelFull);
return StatusCode::SERVICE_UNAVAILABLE;
}
StatusCode::OK
}
const MAX_CONCURRENT_OTLP_HTTP: usize = 32;
async fn memory_pressure_guard(
State(state): State<OtlpHttpState>,
request: axum::extract::Request,
next: axum::middleware::Next,
) -> axum::response::Response {
use axum::response::IntoResponse;
if let Some(m) = state.metrics.as_ref()
&& m.ingest_over_memory_limit()
{
m.record_otlp_reject(OtlpRejectReason::MemoryPressure);
return StatusCode::SERVICE_UNAVAILABLE.into_response();
}
next.run(request).await
}
let state = OtlpHttpState { sink, metrics };
let guard_state = state.clone();
let router = Router::new()
.route("/v1/traces", post(handle_traces))
.route_layer(tower::limit::GlobalConcurrencyLimitLayer::new(
MAX_CONCURRENT_OTLP_HTTP,
))
.with_state(state)
.layer(axum::extract::DefaultBodyLimit::max(max_payload_size));
#[cfg(feature = "daemon")]
let router = router
.layer(tower_http::decompression::RequestDecompressionLayer::new())
.layer(tower_http::limit::RequestBodyLimitLayer::new(
max_payload_size,
));
router.layer(axum::middleware::from_fn_with_state(
guard_state,
memory_pressure_guard,
))
}
#[cfg(test)]
mod tests;