grow_better_stack_logs/
exporter.rs1use std::io::Write;
4use std::sync::Arc;
5
6use async_trait::async_trait;
7use flate2::write::GzEncoder;
8use flate2::Compression;
9use serde_json::Value;
10
11use crate::client::Inner;
12use crate::config::Encoding;
13use crate::core::{Error, Result};
14use crate::encoding::{encode_event, encode_record, frame, FRAME_OVERHEAD};
15use crate::http::map_ingest_error;
16use crate::logging::{ExportOptions, ExportResult, LogExporter, LogRecord};
17
18pub const MAX_REQUEST_BYTES: usize = 10 * 1024 * 1024;
21pub const MAX_RECORD_BYTES: usize = 10 * 1024 * 1024;
23
24#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
26pub struct IngestExporterOptions {
27 pub encoding: Option<Encoding>,
29 pub max_request_bytes: Option<usize>,
31 pub max_record_bytes: Option<usize>,
33}
34
35impl IngestExporterOptions {
36 pub fn encoding(mut self, encoding: Encoding) -> Self {
38 self.encoding = Some(encoding);
39 self
40 }
41 pub fn max_request_bytes(mut self, n: usize) -> Self {
43 self.max_request_bytes = Some(n);
44 self
45 }
46 pub fn max_record_bytes(mut self, n: usize) -> Self {
48 self.max_record_bytes = Some(n);
49 self
50 }
51}
52
53#[derive(Debug, Clone, Copy, PartialEq, Eq)]
55pub struct SendEventsOptions {
56 pub encoding: Option<Encoding>,
58 pub gzip: bool,
60}
61
62impl Default for SendEventsOptions {
63 fn default() -> Self {
64 SendEventsOptions {
65 encoding: None,
66 gzip: true,
67 }
68 }
69}
70
71#[derive(Clone)]
78pub struct BetterStackLogExporter {
79 inner: Arc<Inner>,
80 encoding: Encoding,
81 max_request: usize,
82 max_record: usize,
83}
84
85impl std::fmt::Debug for BetterStackLogExporter {
86 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
87 f.debug_struct("BetterStackLogExporter")
88 .field("encoding", &self.encoding)
89 .field("max_request_bytes", &self.max_request)
90 .field("max_record_bytes", &self.max_record)
91 .finish()
92 }
93}
94
95fn limit(value: Option<usize>, name: &str) -> Result<usize> {
96 match value {
97 None => Ok(MAX_REQUEST_BYTES),
98 Some(n) if (1..=MAX_REQUEST_BYTES).contains(&n) => Ok(n),
99 Some(_) => Err(Error::validation(format!(
100 "logs.exporter: {name} must be between 1 and 10 MiB"
101 ))),
102 }
103}
104
105pub(crate) fn gzip(body: &[u8]) -> Vec<u8> {
106 let mut enc = GzEncoder::new(Vec::with_capacity(body.len() / 4 + 64), Compression::fast());
107 let _ = enc.write_all(body);
109 enc.finish().unwrap_or_default()
110}
111
112impl BetterStackLogExporter {
113 pub(crate) fn new(inner: Arc<Inner>, opts: IngestExporterOptions) -> Result<Self> {
114 let encoding = opts.encoding.unwrap_or(inner.config.encoding());
115 Ok(BetterStackLogExporter {
116 inner,
117 encoding,
118 max_request: limit(opts.max_request_bytes, "max_request_bytes")?,
119 max_record: limit(opts.max_record_bytes, "max_record_bytes")?,
120 })
121 }
122
123 pub fn encoding(&self) -> Encoding {
125 self.encoding
126 }
127
128 pub fn max_request_bytes(&self) -> usize {
130 self.max_request
131 }
132
133 pub fn max_record_bytes(&self) -> usize {
135 self.max_record
136 }
137
138 fn split(&self, encoded: Vec<Vec<u8>>) -> Vec<Vec<Vec<u8>>> {
140 let mut out = Vec::new();
141 let mut cur: Vec<Vec<u8>> = Vec::new();
142 let mut size = FRAME_OVERHEAD;
143 for e in encoded {
144 let n = e.len() + 1;
145 if !cur.is_empty() && size + n > self.max_request {
146 out.push(std::mem::take(&mut cur));
147 size = FRAME_OVERHEAD;
148 }
149 size += n;
150 cur.push(e);
151 }
152 if !cur.is_empty() {
153 out.push(cur);
154 }
155 out
156 }
157
158 pub async fn send_events(&self, events: &[Value], opts: SendEventsOptions) -> Result<()> {
163 const OP: &str = "logs.sendEvents";
164 let encoding = opts.encoding.unwrap_or(self.encoding);
165 let encoded = events
166 .iter()
167 .map(|e| encode_event(e, encoding))
168 .collect::<Result<Vec<_>>>()?;
169 let body = frame(encoding, &encoded);
170 if body.len() > self.max_request {
171 return Err(Error::validation(format!(
172 "{OP}: request body of {} bytes exceeds {} bytes",
173 body.len(),
174 self.max_request
175 )));
176 }
177 self.inner
178 .post_ingest(OP, "/", body, encoding.content_type(), opts.gzip)
179 .await
180 }
181}
182
183#[async_trait]
184impl LogExporter for BetterStackLogExporter {
185 async fn export(&self, records: &[LogRecord], opts: ExportOptions) -> Result<ExportResult> {
189 const OP: &str = "logs.export";
190 self.inner.ingest_transport(OP)?;
191 let mut rejected = 0usize;
192 let mut encoded = Vec::with_capacity(records.len());
193 for r in records {
194 let e = encode_record(r, self.encoding);
195 if e.len() > self.max_record || e.len() + FRAME_OVERHEAD > self.max_request {
196 rejected += 1;
197 } else {
198 encoded.push(e);
199 }
200 }
201 let mut error = (rejected > 0).then(|| {
202 Error::validation(format!(
203 "{OP}: {rejected} record(s) larger than {} bytes dropped",
204 self.max_record.min(self.max_request - FRAME_OVERHEAD)
205 ))
206 });
207 let mut accepted = 0usize;
208 for chunk in self.split(encoded) {
209 let body = frame(self.encoding, &chunk);
210 if let Err(err) = self
211 .inner
212 .post_ingest(OP, "/", body, self.encoding.content_type(), opts.gzip)
213 .await
214 {
215 if accepted == 0 {
216 return Err(err);
217 }
218 return Ok(ExportResult::partial(
219 accepted,
220 records.len() - accepted,
221 Some(err),
222 ));
223 }
224 accepted += chunk.len();
225 }
226 if accepted == 0 {
227 if let Some(err) = error.take() {
228 return Err(err);
229 }
230 }
231 Ok(ExportResult::partial(accepted, rejected, error))
232 }
233}
234
235impl Inner {
236 pub(crate) async fn post_ingest(
238 &self,
239 operation: &str,
240 path: &str,
241 body: Vec<u8>,
242 content_type: &str,
243 gz: bool,
244 ) -> Result<()> {
245 let transport = self.ingest_transport(operation)?;
246 let token = self
247 .config
248 .source_token()
249 .ok_or_else(|| Error::config(format!("{operation}: source_token is required")))?;
250 let mut req = transport
251 .post(path)
252 .operation(operation)
253 .bearer(token)
254 .idempotent(true);
255 if gz {
256 req = req
257 .header("content-encoding", "gzip")
258 .body(gzip(&body), content_type);
259 } else {
260 req = req.body(body, content_type);
261 }
262 req.send().await.map(|_| ()).map_err(map_ingest_error)
263 }
264}