Skip to main content

grow_better_stack_logs/
exporter.rs

1//! The Better Stack ingest exporter: `POST {ingesting host}/` (API.md ยง3).
2
3use std::io::Write;
4use std::sync::Arc;
5
6use async_trait::async_trait;
7use flate2::write::GzEncoder;
8use flate2::Compression;
9use serde_json::Value;
10
11use crate::client::Inner;
12use crate::config::Encoding;
13use crate::core::{Error, Result};
14use crate::encoding::{encode_event, encode_record, frame, FRAME_OVERHEAD};
15use crate::http::map_ingest_error;
16use crate::logging::{ExportOptions, ExportResult, LogExporter, LogRecord};
17
18/// Better Stack limit for one request body (10 MiB, measured compressed). The exporter never
19/// builds a larger uncompressed body, so the limit always holds.
20pub const MAX_REQUEST_BYTES: usize = 10 * 1024 * 1024;
21/// Better Stack limit for one record (10 MiB uncompressed).
22pub const MAX_RECORD_BYTES: usize = 10 * 1024 * 1024;
23
24/// Options of [`BetterStackClient::new_exporter`](crate::BetterStackClient::new_exporter).
25#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
26pub struct IngestExporterOptions {
27    /// Body encoding (default: the config's encoding).
28    pub encoding: Option<Encoding>,
29    /// Uncompressed bytes per request, 1..=10 MiB (default 10 MiB).
30    pub max_request_bytes: Option<usize>,
31    /// Encoded bytes per record, 1..=10 MiB (default 10 MiB); larger records are rejected.
32    pub max_record_bytes: Option<usize>,
33}
34
35impl IngestExporterOptions {
36    /// Sets the encoding.
37    pub fn encoding(mut self, encoding: Encoding) -> Self {
38        self.encoding = Some(encoding);
39        self
40    }
41    /// Sets the request size limit.
42    pub fn max_request_bytes(mut self, n: usize) -> Self {
43        self.max_request_bytes = Some(n);
44        self
45    }
46    /// Sets the record size limit.
47    pub fn max_record_bytes(mut self, n: usize) -> Self {
48        self.max_record_bytes = Some(n);
49        self
50    }
51}
52
53/// Options of [`BetterStackLogExporter::send_events`].
54#[derive(Debug, Clone, Copy, PartialEq, Eq)]
55pub struct SendEventsOptions {
56    /// Body encoding (default: the exporter's encoding).
57    pub encoding: Option<Encoding>,
58    /// Gzip the body (`Content-Encoding: gzip`, default `true`).
59    pub gzip: bool,
60}
61
62impl Default for SendEventsOptions {
63    fn default() -> Self {
64        SendEventsOptions {
65            encoding: None,
66            gzip: true,
67        }
68    }
69}
70
71/// [`LogExporter`] for Better Stack's HTTP ingest API: maps records to events
72/// ([`to_better_stack_event`](crate::to_better_stack_event)), splits batches so no request
73/// exceeds 10 MiB, encodes them as JSON, NDJSON or MessagePack, gzips when asked and posts with
74/// `Authorization: Bearer <source token>`. Exports are retried like idempotent calls (429, 5xx,
75/// network errors); Better Stack does not deduplicate, so an ambiguous failure may store a
76/// record twice.
77#[derive(Clone)]
78pub struct BetterStackLogExporter {
79    inner: Arc<Inner>,
80    encoding: Encoding,
81    max_request: usize,
82    max_record: usize,
83}
84
85impl std::fmt::Debug for BetterStackLogExporter {
86    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
87        f.debug_struct("BetterStackLogExporter")
88            .field("encoding", &self.encoding)
89            .field("max_request_bytes", &self.max_request)
90            .field("max_record_bytes", &self.max_record)
91            .finish()
92    }
93}
94
95fn limit(value: Option<usize>, name: &str) -> Result<usize> {
96    match value {
97        None => Ok(MAX_REQUEST_BYTES),
98        Some(n) if (1..=MAX_REQUEST_BYTES).contains(&n) => Ok(n),
99        Some(_) => Err(Error::validation(format!(
100            "logs.exporter: {name} must be between 1 and 10 MiB"
101        ))),
102    }
103}
104
105pub(crate) fn gzip(body: &[u8]) -> Vec<u8> {
106    let mut enc = GzEncoder::new(Vec::with_capacity(body.len() / 4 + 64), Compression::fast());
107    // Writing into a Vec cannot fail.
108    let _ = enc.write_all(body);
109    enc.finish().unwrap_or_default()
110}
111
112impl BetterStackLogExporter {
113    pub(crate) fn new(inner: Arc<Inner>, opts: IngestExporterOptions) -> Result<Self> {
114        let encoding = opts.encoding.unwrap_or(inner.config.encoding());
115        Ok(BetterStackLogExporter {
116            inner,
117            encoding,
118            max_request: limit(opts.max_request_bytes, "max_request_bytes")?,
119            max_record: limit(opts.max_record_bytes, "max_record_bytes")?,
120        })
121    }
122
123    /// Body encoding of [`export`](LogExporter::export).
124    pub fn encoding(&self) -> Encoding {
125        self.encoding
126    }
127
128    /// Uncompressed size limit of one request.
129    pub fn max_request_bytes(&self) -> usize {
130        self.max_request
131    }
132
133    /// Encoded size limit of one record.
134    pub fn max_record_bytes(&self) -> usize {
135        self.max_record
136    }
137
138    /// Splits encoded records into request chunks that stay within the request limit.
139    fn split(&self, encoded: Vec<Vec<u8>>) -> Vec<Vec<Vec<u8>>> {
140        let mut out = Vec::new();
141        let mut cur: Vec<Vec<u8>> = Vec::new();
142        let mut size = FRAME_OVERHEAD;
143        for e in encoded {
144            let n = e.len() + 1;
145            if !cur.is_empty() && size + n > self.max_request {
146                out.push(std::mem::take(&mut cur));
147                size = FRAME_OVERHEAD;
148            }
149            size += n;
150            cur.push(e);
151        }
152        if !cur.is_empty() {
153            out.push(cur);
154        }
155        out
156    }
157
158    /// Posts raw events (JSON objects; `dt` may be an RFC 3339 string or UNIX seconds/ms/ns) as
159    /// one request, without splitting. Errors: 403 โ†’ `AUTHENTICATION`, 402 โ†’ `RATE_LIMITED` (not
160    /// retryable), 406/413 โ†’ `VALIDATION`; a body over the request limit is a `VALIDATION`
161    /// error before anything is sent.
162    pub async fn send_events(&self, events: &[Value], opts: SendEventsOptions) -> Result<()> {
163        const OP: &str = "logs.sendEvents";
164        let encoding = opts.encoding.unwrap_or(self.encoding);
165        let encoded = events
166            .iter()
167            .map(|e| encode_event(e, encoding))
168            .collect::<Result<Vec<_>>>()?;
169        let body = frame(encoding, &encoded);
170        if body.len() > self.max_request {
171            return Err(Error::validation(format!(
172                "{OP}: request body of {} bytes exceeds {} bytes",
173                body.len(),
174                self.max_request
175            )));
176        }
177        self.inner
178            .post_ingest(OP, "/", body, encoding.content_type(), opts.gzip)
179            .await
180    }
181}
182
183#[async_trait]
184impl LogExporter for BetterStackLogExporter {
185    /// Encodes and posts records. Records over the record limit are rejected with a
186    /// `VALIDATION` error before anything is sent; a failed request fails its records and the
187    /// rest of the batch. Returns `Err` when nothing was accepted.
188    async fn export(&self, records: &[LogRecord], opts: ExportOptions) -> Result<ExportResult> {
189        const OP: &str = "logs.export";
190        self.inner.ingest_transport(OP)?;
191        let mut rejected = 0usize;
192        let mut encoded = Vec::with_capacity(records.len());
193        for r in records {
194            let e = encode_record(r, self.encoding);
195            if e.len() > self.max_record || e.len() + FRAME_OVERHEAD > self.max_request {
196                rejected += 1;
197            } else {
198                encoded.push(e);
199            }
200        }
201        let mut error = (rejected > 0).then(|| {
202            Error::validation(format!(
203                "{OP}: {rejected} record(s) larger than {} bytes dropped",
204                self.max_record.min(self.max_request - FRAME_OVERHEAD)
205            ))
206        });
207        let mut accepted = 0usize;
208        for chunk in self.split(encoded) {
209            let body = frame(self.encoding, &chunk);
210            if let Err(err) = self
211                .inner
212                .post_ingest(OP, "/", body, self.encoding.content_type(), opts.gzip)
213                .await
214            {
215                if accepted == 0 {
216                    return Err(err);
217                }
218                return Ok(ExportResult::partial(
219                    accepted,
220                    records.len() - accepted,
221                    Some(err),
222                ));
223            }
224            accepted += chunk.len();
225        }
226        if accepted == 0 {
227            if let Some(err) = error.take() {
228                return Err(err);
229            }
230        }
231        Ok(ExportResult::partial(accepted, rejected, error))
232    }
233}
234
235impl Inner {
236    /// Sends one ingest request body (optionally gzipped) with the source token.
237    pub(crate) async fn post_ingest(
238        &self,
239        operation: &str,
240        path: &str,
241        body: Vec<u8>,
242        content_type: &str,
243        gz: bool,
244    ) -> Result<()> {
245        let transport = self.ingest_transport(operation)?;
246        let token = self
247            .config
248            .source_token()
249            .ok_or_else(|| Error::config(format!("{operation}: source_token is required")))?;
250        let mut req = transport
251            .post(path)
252            .operation(operation)
253            .bearer(token)
254            .idempotent(true);
255        if gz {
256            req = req
257                .header("content-encoding", "gzip")
258                .body(gzip(&body), content_type);
259        } else {
260            req = req.body(body, content_type);
261        }
262        req.send().await.map(|_| ()).map_err(map_ingest_error)
263    }
264}