Skip to main content

graphrt_otel/
http.rs

1//! OTLP/HTTP JSON exporter with batching, a bounded queue and retries.
2//!
3//! `export` never blocks: spans go into a bounded channel and are dropped (and
4//! counted) when it is full. A background tokio task batches spans by size and
5//! interval, POSTs them to `.../v1/traces`, retries 429/502/503/504 and network
6//! errors with backoff (honoring `Retry-After`), and periodically pushes metrics.
7
8use std::sync::Arc;
9use std::sync::atomic::{AtomicU64, Ordering};
10use std::time::Duration;
11
12use futures::future::BoxFuture;
13use tokio::sync::{mpsc, oneshot};
14
15use crate::config::OtelConfig;
16use crate::exporter::SpanExporter;
17use crate::metrics::Metrics;
18use crate::otlp::{self, Resource};
19use crate::span::Span;
20
21/// Exporter construction error.
22#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
23#[non_exhaustive]
24pub enum ExportError {
25    #[error("the OTLP exporter needs a running tokio runtime")]
26    NoRuntime,
27    #[error("cannot build HTTP client: {0}")]
28    Client(String),
29}
30
31/// Tuning knobs.
32#[derive(Clone, Debug)]
33pub struct ExportSettings {
34    /// Spans per request.
35    pub batch_size: usize,
36    /// Spans buffered before new ones are dropped.
37    pub queue_capacity: usize,
38    /// Maximum delay before a partial batch is sent.
39    pub flush_interval: Duration,
40    /// Interval between metric pushes.
41    pub metrics_interval: Duration,
42    /// Per-request timeout.
43    pub timeout: Duration,
44    /// Retries after the first attempt for retryable failures.
45    pub max_retries: u32,
46    /// First retry delay (doubled each retry, capped at 30 s).
47    pub initial_backoff: Duration,
48}
49
50impl Default for ExportSettings {
51    fn default() -> Self {
52        Self {
53            batch_size: 512,
54            queue_capacity: 4096,
55            flush_interval: Duration::from_secs(2),
56            metrics_interval: Duration::from_secs(30),
57            timeout: Duration::from_secs(10),
58            max_retries: 3,
59            initial_backoff: Duration::from_millis(500),
60        }
61    }
62}
63
64enum Msg {
65    Span(Box<Span>),
66    Flush(oneshot::Sender<()>),
67    Shutdown(oneshot::Sender<()>),
68}
69
70#[derive(Debug, Default)]
71struct Counters {
72    dropped: AtomicU64,
73    exported: AtomicU64,
74    failed: AtomicU64,
75}
76
77/// OTLP/HTTP JSON span (and metric) exporter.
78#[derive(Debug)]
79pub struct OtlpHttpExporter {
80    tx: mpsc::Sender<Msg>,
81    counters: Arc<Counters>,
82}
83
84impl OtlpHttpExporter {
85    /// Start the exporter (spawns its batching task on the current tokio runtime).
86    pub fn start(config: &OtelConfig, metrics: Option<Arc<Metrics>>) -> Result<Self, ExportError> {
87        Self::start_with(config, metrics, ExportSettings::default())
88    }
89
90    /// [`OtlpHttpExporter::start`] with explicit settings.
91    pub fn start_with(
92        config: &OtelConfig,
93        metrics: Option<Arc<Metrics>>,
94        settings: ExportSettings,
95    ) -> Result<Self, ExportError> {
96        let handle = tokio::runtime::Handle::try_current().map_err(|_| ExportError::NoRuntime)?;
97        install_crypto_provider();
98        let client = reqwest::Client::builder()
99            .timeout(settings.timeout)
100            .no_proxy()
101            .build()
102            .map_err(|e| ExportError::Client(e.without_url().to_string()))?;
103        let (tx, rx) = mpsc::channel(settings.queue_capacity.max(1));
104        let counters = Arc::new(Counters::default());
105        let worker = Worker {
106            client,
107            traces_url: config.traces_endpoint.clone(),
108            metrics_url: config.metrics_endpoint.clone(),
109            headers: config.headers.clone(),
110            resource: config.resource.clone(),
111            metrics,
112            settings,
113            counters: counters.clone(),
114        };
115        handle.spawn(worker.run(rx));
116        Ok(Self { tx, counters })
117    }
118
119    /// Spans dropped because the queue was full or the exporter stopped.
120    pub fn dropped(&self) -> u64 {
121        self.counters.dropped.load(Ordering::Relaxed)
122    }
123
124    /// Spans delivered successfully.
125    pub fn exported(&self) -> u64 {
126        self.counters.exported.load(Ordering::Relaxed)
127    }
128
129    /// Spans lost after exhausting retries or on non-retryable errors.
130    pub fn failed(&self) -> u64 {
131        self.counters.failed.load(Ordering::Relaxed)
132    }
133
134    /// Send everything queued, push metrics, and stop the background task.
135    pub async fn shutdown(&self) {
136        let (done, wait) = oneshot::channel();
137        if self.tx.send(Msg::Shutdown(done)).await.is_ok() {
138            let _closed = wait.await;
139        }
140    }
141}
142
143impl SpanExporter for OtlpHttpExporter {
144    fn export(&self, span: Span) {
145        if self.tx.try_send(Msg::Span(Box::new(span))).is_err() {
146            self.counters.dropped.fetch_add(1, Ordering::Relaxed);
147        }
148    }
149
150    fn flush(&self) -> BoxFuture<'_, ()> {
151        Box::pin(async move {
152            let (done, wait) = oneshot::channel();
153            if self.tx.send(Msg::Flush(done)).await.is_ok() {
154                let _closed = wait.await;
155            }
156        })
157    }
158}
159
160struct Worker {
161    client: reqwest::Client,
162    traces_url: String,
163    metrics_url: String,
164    headers: Vec<(String, String)>,
165    resource: Resource,
166    metrics: Option<Arc<Metrics>>,
167    settings: ExportSettings,
168    counters: Arc<Counters>,
169}
170
171impl Worker {
172    async fn run(self, mut rx: mpsc::Receiver<Msg>) {
173        let mut batch: Vec<Span> = Vec::new();
174        // `interval_at` skips the immediate first tick `interval` would fire.
175        let start = tokio::time::Instant::now();
176        let mut flush_tick =
177            tokio::time::interval_at(start + self.settings.flush_interval, self.settings.flush_interval);
178        let mut metrics_tick =
179            tokio::time::interval_at(start + self.settings.metrics_interval, self.settings.metrics_interval);
180        flush_tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
181        metrics_tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
182        loop {
183            tokio::select! {
184                msg = rx.recv() => match msg {
185                    Some(Msg::Span(span)) => {
186                        batch.push(*span);
187                        if batch.len() >= self.settings.batch_size {
188                            self.send_spans(std::mem::take(&mut batch)).await;
189                        }
190                    }
191                    Some(Msg::Flush(done)) => {
192                        self.send_spans(std::mem::take(&mut batch)).await;
193                        self.send_metrics().await;
194                        let _receiver_gone = done.send(());
195                    }
196                    Some(Msg::Shutdown(done)) => {
197                        rx.close();
198                        while let Ok(Msg::Span(span)) = rx.try_recv() {
199                            batch.push(*span);
200                        }
201                        self.send_spans(std::mem::take(&mut batch)).await;
202                        self.send_metrics().await;
203                        let _receiver_gone = done.send(());
204                        return;
205                    }
206                    None => {
207                        self.send_spans(std::mem::take(&mut batch)).await;
208                        self.send_metrics().await;
209                        return;
210                    }
211                },
212                _ = flush_tick.tick() => {
213                    if !batch.is_empty() {
214                        self.send_spans(std::mem::take(&mut batch)).await;
215                    }
216                }
217                _ = metrics_tick.tick() => self.send_metrics().await,
218            }
219        }
220    }
221
222    async fn send_spans(&self, spans: Vec<Span>) {
223        if spans.is_empty() {
224            return;
225        }
226        for chunk in spans.chunks(self.settings.batch_size.max(1)) {
227            let body = otlp::traces_request(&self.resource, chunk);
228            let n = u64::try_from(chunk.len()).unwrap_or(u64::MAX);
229            if self.post(&self.traces_url, &body).await {
230                self.counters.exported.fetch_add(n, Ordering::Relaxed);
231            } else {
232                self.counters.failed.fetch_add(n, Ordering::Relaxed);
233            }
234        }
235    }
236
237    async fn send_metrics(&self) {
238        let Some(metrics) = &self.metrics else { return };
239        let encoded = metrics.encode();
240        if encoded.is_empty() {
241            return;
242        }
243        let body = otlp::metrics_request(&self.resource, encoded);
244        // Metrics are cumulative: a failed push is superseded by the next one.
245        let _delivered = self.post(&self.metrics_url, &body).await;
246    }
247
248    /// POST with retries. Returns whether the collector accepted the request.
249    async fn post(&self, url: &str, body: &serde_json::Value) -> bool {
250        let Ok(bytes) = serde_json::to_vec(body) else { return false };
251        let mut delay = self.settings.initial_backoff;
252        for attempt in 0..=self.settings.max_retries {
253            let mut request = self.client.post(url).header("content-type", "application/json").body(bytes.clone());
254            for (k, v) in &self.headers {
255                request = request.header(k.as_str(), v.as_str());
256            }
257            let retry_after = match request.send().await {
258                Ok(response) if response.status().is_success() => return true,
259                Ok(response) => {
260                    let status = response.status().as_u16();
261                    if !matches!(status, 429 | 502 | 503 | 504) {
262                        return false;
263                    }
264                    response
265                        .headers()
266                        .get("retry-after")
267                        .and_then(|v| v.to_str().ok())
268                        .and_then(|v| v.trim().parse::<u64>().ok())
269                        .map(Duration::from_secs)
270                }
271                Err(_) => None,
272            };
273            if attempt == self.settings.max_retries {
274                break;
275            }
276            let wait = retry_after.unwrap_or(delay).min(Duration::from_secs(30));
277            tokio::time::sleep(wait).await;
278            delay = (delay * 2).min(Duration::from_secs(30));
279        }
280        false
281    }
282}
283
284fn install_crypto_provider() {
285    static ONCE: std::sync::Once = std::sync::Once::new();
286    ONCE.call_once(|| {
287        let _already_installed = rustls::crypto::ring::default_provider().install_default();
288    });
289}