Skip to main content

grow_graylog/
gelf_transport.rs

1//! GELF transports: [`GelfUdpTransport`] (chunked, compressed datagrams),
2//! [`GelfTcpTransport`] (NUL-framed, TLS, reconnect) and [`GelfHttpTransport`] (bulk, gzip).
3
4use std::fmt;
5use std::io::Write as _;
6use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
7use std::sync::Arc;
8use std::time::Duration;
9
10use async_trait::async_trait;
11use flate2::write::{GzEncoder, ZlibEncoder};
12use flate2::Compression;
13use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
14use tokio::net::{TcpStream, UdpSocket};
15use tokio::sync::Mutex;
16
17use crate::client::HEADER_REQUESTED_BY;
18use crate::config::{url_host, GelfProtocol, DEFAULT_GELF_PORT, DEFAULT_REQUESTED_BY};
19use crate::core::config::{DEFAULT_CONNECT_TIMEOUT, DEFAULT_REQUEST_TIMEOUT};
20use crate::core::redact::is_sensitive_header;
21use crate::core::{CommonConfig, Error, HttpTransport, Result, RetryPolicy, Secret, TlsConfig};
22use crate::gelf::GelfMessage;
23
24/// Per-call options of [`GelfTransport::send`].
25#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
26pub struct GelfSendOptions {
27    /// Overrides the HTTP transport's compression for this call (`None` = transport default).
28    /// UDP and TCP ignore it.
29    pub gzip: Option<bool>,
30}
31
32/// Outcome of [`GelfTransport::send`].
33#[derive(Debug, Default)]
34pub struct GelfSendResult {
35    /// Messages handed to the network.
36    pub sent: usize,
37    /// Messages rejected client-side (for example too large for UDP).
38    pub rejected: usize,
39    /// Why messages were rejected.
40    pub error: Option<Error>,
41}
42
43/// Sends GELF messages to a Graylog input. Implementations are safe for concurrent use.
44#[async_trait]
45pub trait GelfTransport: Send + Sync + fmt::Debug {
46    /// `udp`, `tcp` or `http`.
47    fn protocol(&self) -> GelfProtocol;
48
49    /// Delivers messages in order. Network failures are errors (`UNAVAILABLE`, `TIMEOUT`);
50    /// client-side rejections are counted in the result. A closed transport returns `CONFIG`.
51    async fn send(&self, messages: &[GelfMessage], opts: GelfSendOptions)
52        -> Result<GelfSendResult>;
53
54    /// Releases sockets. Idempotent.
55    async fn close(&self) -> Result<()>;
56}
57
58/// Payload compression of the UDP transport.
59#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
60pub enum GelfCompression {
61    /// gzip (default).
62    #[default]
63    Gzip,
64    /// zlib.
65    Zlib,
66    /// No compression.
67    None,
68}
69
70/// Chunk magic bytes `0x1e 0x0f`.
71pub const GELF_CHUNK_MAGIC: [u8; 2] = [0x1e, 0x0f];
72/// Chunk header: magic (2), message id (8), sequence number (1), sequence count (1).
73pub const GELF_CHUNK_HEADER_SIZE: usize = 12;
74/// GELF limit of chunks per message.
75pub const GELF_MAX_CHUNKS: usize = 128;
76/// Default datagram size including the chunk header (fits a 1500-byte Ethernet MTU).
77pub const DEFAULT_UDP_CHUNK_SIZE: usize = 1420;
78
79fn check_host_port(op: &str, host: &str, port: u16) -> Result<String> {
80    if host.trim().is_empty() {
81        return Err(Error::config(format!("{op}: host must not be blank")));
82    }
83    if port == 0 {
84        return Err(Error::config(format!(
85            "{op}: port must be an integer 1..65535"
86        )));
87    }
88    Ok(format!("{}:{port}", url_host(host)))
89}
90
91fn network_error(op: &str, err: &std::io::Error) -> Error {
92    if err.kind() == std::io::ErrorKind::TimedOut {
93        return Error::timeout(format!("{op}: timeout"));
94    }
95    Error::unavailable(format!("{op}: network error: {}", err.kind()))
96}
97
98fn closed_error(op: &str) -> Error {
99    Error::config(format!("{op}: transport is closed"))
100}
101
102/// Compresses `payload` (gzip, zlib or not at all).
103pub(crate) fn compress(compression: GelfCompression, payload: &[u8]) -> Vec<u8> {
104    match compression {
105        GelfCompression::Gzip => {
106            let mut e = GzEncoder::new(Vec::new(), Compression::default());
107            let _ = e.write_all(payload);
108            e.finish().unwrap_or_default()
109        }
110        GelfCompression::Zlib => {
111            let mut e = ZlibEncoder::new(Vec::new(), Compression::default());
112            let _ = e.write_all(payload);
113            e.finish().unwrap_or_default()
114        }
115        GelfCompression::None => payload.to_vec(),
116    }
117}
118
119/// Compresses `payload` and splits it into datagrams: one datagram when it fits `chunk_size`,
120/// else chunks `0x1e 0x0f | id(8) | seq | count | data`. More than 128 chunks is a
121/// `VALIDATION` error. `message_id = None` uses a random id.
122pub fn encode_gelf_udp(
123    payload: &[u8],
124    compression: GelfCompression,
125    chunk_size: usize,
126    message_id: Option<[u8; 8]>,
127) -> Result<Vec<Vec<u8>>> {
128    if chunk_size <= GELF_CHUNK_HEADER_SIZE {
129        return Err(Error::validation(
130            "gelf.udp: chunk size must exceed the 12-byte header",
131        ));
132    }
133    let data = compress(compression, payload);
134    if data.len() <= chunk_size {
135        return Ok(vec![data]);
136    }
137    let per = chunk_size - GELF_CHUNK_HEADER_SIZE;
138    let count = data.len().div_ceil(per);
139    if count > GELF_MAX_CHUNKS {
140        return Err(Error::validation(format!(
141            "gelf.udp: message needs {count} chunks (max 128); use TCP or HTTP for large messages"
142        )));
143    }
144    let id = message_id.unwrap_or_else(rand::random);
145    Ok(data
146        .chunks(per)
147        .enumerate()
148        .map(|(seq, part)| {
149            let mut chunk = Vec::with_capacity(GELF_CHUNK_HEADER_SIZE + part.len());
150            chunk.extend_from_slice(&GELF_CHUNK_MAGIC);
151            chunk.extend_from_slice(&id);
152            chunk.push(seq as u8);
153            chunk.push(count as u8);
154            chunk.extend_from_slice(part);
155            chunk
156        })
157        .collect())
158}
159
160// ---------------------------------------------------------------------------------------------
161// UDP
162
163/// Options of [`GelfUdpTransport::new`].
164#[derive(Debug, Clone, PartialEq, Eq)]
165pub struct GelfUdpOptions {
166    /// Host of the GELF UDP input (required).
167    pub host: String,
168    /// Port (default 12201).
169    pub port: u16,
170    /// Compression applied before chunking (default gzip).
171    pub compression: GelfCompression,
172    /// Datagram size including the 12-byte chunk header (64..=65507, default 1420).
173    pub chunk_size: usize,
174}
175
176impl GelfUdpOptions {
177    /// Options for `host` with the defaults.
178    pub fn new(host: impl Into<String>) -> Self {
179        GelfUdpOptions {
180            host: host.into(),
181            port: DEFAULT_GELF_PORT,
182            compression: GelfCompression::Gzip,
183            chunk_size: DEFAULT_UDP_CHUNK_SIZE,
184        }
185    }
186}
187
188/// Fire-and-forget datagrams: compressed, chunked up to 128 chunks. Delivery is best-effort
189/// (UDP has no acknowledgement).
190#[derive(Debug)]
191pub struct GelfUdpTransport {
192    addr: String,
193    compression: GelfCompression,
194    chunk_size: usize,
195    socket: Mutex<Option<UdpSocket>>,
196    closed: AtomicBool,
197}
198
199impl GelfUdpTransport {
200    /// Validates `opts` (`CONFIG`); the socket is opened on first use.
201    pub fn new(opts: GelfUdpOptions) -> Result<Self> {
202        let addr = check_host_port("gelf.udp", &opts.host, opts.port)?;
203        if !(64..=65507).contains(&opts.chunk_size) {
204            return Err(Error::config(
205                "gelf.udp: chunkSize must be an integer 64..65507",
206            ));
207        }
208        Ok(GelfUdpTransport {
209            addr,
210            compression: opts.compression,
211            chunk_size: opts.chunk_size,
212            socket: Mutex::new(None),
213            closed: AtomicBool::new(false),
214        })
215    }
216}
217
218#[async_trait]
219impl GelfTransport for GelfUdpTransport {
220    fn protocol(&self) -> GelfProtocol {
221        GelfProtocol::Udp
222    }
223
224    /// Writes every message as one or more datagrams. Messages that need more than 128 chunks
225    /// are rejected (counted, not sent).
226    async fn send(&self, messages: &[GelfMessage], _: GelfSendOptions) -> Result<GelfSendResult> {
227        let mut guard = self.socket.lock().await;
228        if self.closed.load(Ordering::SeqCst) {
229            return Err(closed_error("gelf.udp"));
230        }
231        let mut res = GelfSendResult::default();
232        for m in messages {
233            let datagrams =
234                match encode_gelf_udp(&m.encode(), self.compression, self.chunk_size, None) {
235                    Ok(d) => d,
236                    Err(e) => {
237                        res.rejected += 1;
238                        res.error = Some(e);
239                        continue;
240                    }
241                };
242            if guard.is_none() {
243                let bind = if self.addr.starts_with('[') {
244                    "[::]:0"
245                } else {
246                    "0.0.0.0:0"
247                };
248                let sock = UdpSocket::bind(bind)
249                    .await
250                    .map_err(|e| network_error("gelf.udp", &e))?;
251                sock.connect(&self.addr)
252                    .await
253                    .map_err(|e| network_error("gelf.udp", &e))?;
254                *guard = Some(sock);
255            }
256            let sock = guard.as_ref().expect("socket opened above");
257            for d in &datagrams {
258                if let Err(e) = sock.send(d).await {
259                    *guard = None;
260                    return Err(network_error("gelf.udp", &e));
261                }
262            }
263            res.sent += 1;
264        }
265        Ok(res)
266    }
267
268    async fn close(&self) -> Result<()> {
269        let mut guard = self.socket.lock().await;
270        self.closed.store(true, Ordering::SeqCst);
271        *guard = None;
272        Ok(())
273    }
274}
275
276// ---------------------------------------------------------------------------------------------
277// TCP
278
279/// Options of [`GelfTcpTransport::new`].
280#[derive(Debug, Clone)]
281pub struct GelfTcpOptions {
282    /// Host of the GELF TCP input (required).
283    pub host: String,
284    /// Port (default 12201).
285    pub port: u16,
286    /// Enables TLS with `tls_config` (the input needs `tls_enable`).
287    pub tls: bool,
288    /// TLS settings (CA, client certificate, `insecure_skip_verify`).
289    pub tls_config: TlsConfig,
290    /// SNI / verification name (default `host`).
291    pub server_name: Option<String>,
292    /// Connection setup timeout (default 10 s).
293    pub connect_timeout: Duration,
294    /// Write timeout per batch (default 30 s).
295    pub write_timeout: Duration,
296    /// Reconnects: a batch whose connect or write fails is re-sent on a new connection up to
297    /// `retry.max_retries` times with full-jitter backoff.
298    pub retry: RetryPolicy,
299}
300
301impl GelfTcpOptions {
302    /// Options for `host` with the defaults.
303    pub fn new(host: impl Into<String>) -> Self {
304        GelfTcpOptions {
305            host: host.into(),
306            port: DEFAULT_GELF_PORT,
307            tls: false,
308            tls_config: TlsConfig::default(),
309            server_name: None,
310            connect_timeout: DEFAULT_CONNECT_TIMEOUT,
311            write_timeout: DEFAULT_REQUEST_TIMEOUT,
312            retry: RetryPolicy::default(),
313        }
314    }
315}
316
317trait Conn: AsyncRead + AsyncWrite + Unpin + Send {}
318impl<T: AsyncRead + AsyncWrite + Unpin + Send> Conn for T {}
319
320/// NUL-terminated JSON frames over one long-lived connection (input option
321/// `use_null_delimiter`). Batches never interleave. A connection closed by the server is
322/// detected before each batch and re-opened; a failed batch is re-sent on a new connection
323/// (at-least-once).
324pub struct GelfTcpTransport {
325    addr: String,
326    opts: GelfTcpOptions,
327    tls: Option<(
328        Arc<rustls::ClientConfig>,
329        rustls::pki_types::ServerName<'static>,
330    )>,
331    conn: Mutex<Option<Box<dyn Conn>>>,
332    closed: AtomicBool,
333    connections: AtomicUsize,
334}
335
336impl fmt::Debug for GelfTcpTransport {
337    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
338        f.debug_struct("GelfTcpTransport")
339            .field("addr", &self.addr)
340            .field("secure", &self.secure())
341            .field("connections", &self.connections())
342            .finish_non_exhaustive()
343    }
344}
345
346/// Encodes messages as NUL-terminated JSON frames.
347pub fn frame_gelf_tcp(messages: &[GelfMessage]) -> Vec<u8> {
348    let mut out = Vec::new();
349    for m in messages {
350        out.extend(m.encode());
351        out.push(0);
352    }
353    out
354}
355
356impl GelfTcpTransport {
357    /// Validates `opts` (`CONFIG`); the connection is opened on first use.
358    pub fn new(opts: GelfTcpOptions) -> Result<Self> {
359        let addr = check_host_port("gelf.tcp", &opts.host, opts.port)?;
360        if opts.connect_timeout.is_zero() || opts.write_timeout.is_zero() {
361            return Err(Error::config("gelf.tcp: timeouts must be positive"));
362        }
363        opts.retry.validate()?;
364        let tls = if opts.tls {
365            let name = opts
366                .server_name
367                .clone()
368                .filter(|s| !s.is_empty())
369                .unwrap_or_else(|| opts.host.trim_matches(['[', ']']).to_owned());
370            Some((
371                crate::tls::client_config(&opts.tls_config)?,
372                crate::tls::server_name(&name)?,
373            ))
374        } else {
375            None
376        };
377        Ok(GelfTcpTransport {
378            addr,
379            opts,
380            tls,
381            conn: Mutex::new(None),
382            closed: AtomicBool::new(false),
383            connections: AtomicUsize::new(0),
384        })
385    }
386
387    /// Whether the transport uses TLS.
388    pub fn secure(&self) -> bool {
389        self.tls.is_some()
390    }
391
392    /// Connections opened so far (diagnostics).
393    pub fn connections(&self) -> usize {
394        self.connections.load(Ordering::SeqCst)
395    }
396
397    async fn dial(&self) -> std::io::Result<Box<dyn Conn>> {
398        let connect = async {
399            let tcp = TcpStream::connect(&self.addr).await?;
400            let _ = tcp.set_nodelay(true);
401            match &self.tls {
402                None => Ok(Box::new(tcp) as Box<dyn Conn>),
403                Some((cfg, name)) => {
404                    let tls = tokio_rustls::TlsConnector::from(cfg.clone())
405                        .connect(name.clone(), tcp)
406                        .await?;
407                    Ok(Box::new(tls) as Box<dyn Conn>)
408                }
409            }
410        };
411        match tokio::time::timeout(self.opts.connect_timeout, connect).await {
412            Ok(r) => r,
413            Err(_) => Err(std::io::Error::new(
414                std::io::ErrorKind::TimedOut,
415                "connect timeout",
416            )),
417        }
418    }
419
420    /// Graylog never writes on a GELF TCP connection, so a read that completes means EOF or a
421    /// reset (or unexpected data, which is ignored).
422    async fn alive(conn: &mut Box<dyn Conn>) -> bool {
423        let mut buf = [0u8; 64];
424        match tokio::time::timeout(Duration::from_millis(1), conn.read(&mut buf)).await {
425            Err(_) => true,
426            Ok(Ok(n)) => n > 0,
427            Ok(Err(_)) => false,
428        }
429    }
430
431    async fn write(&self, slot: &mut Option<Box<dyn Conn>>, data: &[u8]) -> std::io::Result<()> {
432        if let Some(c) = slot.as_mut() {
433            if !Self::alive(c).await {
434                *slot = None;
435            }
436        }
437        if slot.is_none() {
438            *slot = Some(self.dial().await?);
439            self.connections.fetch_add(1, Ordering::SeqCst);
440        }
441        let conn = slot.as_mut().expect("connection opened above");
442        let io = async {
443            conn.write_all(data).await?;
444            conn.flush().await
445        };
446        let res = match tokio::time::timeout(self.opts.write_timeout, io).await {
447            Ok(r) => r,
448            Err(_) => Err(std::io::Error::new(
449                std::io::ErrorKind::TimedOut,
450                "write timeout",
451            )),
452        };
453        if res.is_err() {
454            *slot = None;
455        }
456        res
457    }
458}
459
460#[async_trait]
461impl GelfTransport for GelfTcpTransport {
462    fn protocol(&self) -> GelfProtocol {
463        GelfProtocol::Tcp
464    }
465
466    /// Writes all messages as NUL-terminated frames in one write; failed batches are re-sent on
467    /// a new connection (see [`GelfTcpOptions::retry`]).
468    async fn send(&self, messages: &[GelfMessage], _: GelfSendOptions) -> Result<GelfSendResult> {
469        let mut slot = self.conn.lock().await;
470        if self.closed.load(Ordering::SeqCst) {
471            return Err(closed_error("gelf.tcp"));
472        }
473        if messages.is_empty() {
474            return Ok(GelfSendResult::default());
475        }
476        let data = frame_gelf_tcp(messages);
477        let mut attempt = 0;
478        loop {
479            match self.write(&mut slot, &data).await {
480                Ok(()) => {
481                    return Ok(GelfSendResult {
482                        sent: messages.len(),
483                        ..GelfSendResult::default()
484                    })
485                }
486                Err(e) if attempt >= self.opts.retry.max_retries => {
487                    return Err(network_error("gelf.tcp", &e))
488                }
489                Err(_) => {
490                    tokio::time::sleep(self.opts.retry.backoff(attempt)).await;
491                    attempt += 1;
492                }
493            }
494        }
495    }
496
497    async fn close(&self) -> Result<()> {
498        let mut slot = self.conn.lock().await;
499        self.closed.store(true, Ordering::SeqCst);
500        if let Some(mut c) = slot.take() {
501            let _ = c.shutdown().await;
502        }
503        Ok(())
504    }
505}
506
507// ---------------------------------------------------------------------------------------------
508// HTTP
509
510/// Options of [`GelfHttpTransport::new`].
511#[derive(Debug, Clone)]
512pub struct GelfHttpOptions {
513    /// URL of the GELF HTTP input, for example `http://graylog:12202/gelf`.
514    pub url: String,
515    /// gzip-compressed bodies (default true).
516    pub gzip: bool,
517    /// One body of newline-terminated messages per batch (default true); `false` posts one
518    /// request per message (inputs without `enable_bulk_receiving`).
519    pub bulk: bool,
520    /// Extra request headers (for example the input's optional authorization header);
521    /// sensitive values are redacted.
522    pub headers: Vec<(String, String)>,
523    /// Timeouts, retries, TLS, proxy and hooks (its URL and headers are ignored; `None` = the
524    /// defaults).
525    pub config: Option<CommonConfig>,
526}
527
528impl GelfHttpOptions {
529    /// Options for `url` with the defaults.
530    pub fn new(url: impl Into<String>) -> Self {
531        GelfHttpOptions {
532            url: url.into(),
533            gzip: true,
534            bulk: true,
535            headers: Vec::new(),
536            config: None,
537        }
538    }
539}
540
541/// Posts GELF messages to a GELF HTTP input (202 Accepted). Graylog answers 202 even for
542/// invalid JSON or a missing `short_message` and drops the message, so messages are validated
543/// when they are built.
544#[derive(Debug)]
545pub struct GelfHttpTransport {
546    path: String,
547    transport: HttpTransport,
548    gzip: bool,
549    bulk: bool,
550    headers: Vec<(String, Secret)>,
551}
552
553/// A copy of `base` with another URL (and no extra headers).
554pub(crate) fn rebase_config(base: Option<&CommonConfig>, url: &str) -> Result<CommonConfig> {
555    let mut b = CommonConfig::builder().url(url);
556    if let Some(c) = base {
557        b = b
558            .connect_timeout(c.connect_timeout())
559            .request_timeout(c.request_timeout())
560            .total_timeout(c.total_timeout())
561            .retry(c.retry().clone())
562            .tls(c.tls().clone());
563        if let Some(p) = c.proxy() {
564            b = b.proxy(p);
565        }
566        if let Some(s) = c.user_agent_suffix() {
567            b = b.user_agent_suffix(s);
568        }
569        if let Some(h) = c.hooks() {
570            b = b.hooks(h.clone());
571        }
572    }
573    b.build(url)
574}
575
576impl GelfHttpTransport {
577    /// Validates `opts` (`CONFIG`).
578    pub fn new(opts: GelfHttpOptions) -> Result<Self> {
579        let u = url::Url::parse(&opts.url)
580            .ok()
581            .filter(|u| matches!(u.scheme(), "http" | "https") && u.host_str().is_some())
582            .ok_or_else(|| Error::config("gelf.http: url must be an http(s) URL"))?;
583        let base = format!(
584            "{}://{}",
585            u.scheme(),
586            &u[url::Position::BeforeHost..url::Position::AfterPort]
587        );
588        let cfg = rebase_config(opts.config.as_ref(), &base)?;
589        let transport = HttpTransport::new(&cfg)?;
590        let mut headers = Vec::new();
591        for (k, v) in opts.headers {
592            let s = Secret::new(v);
593            if is_sensitive_header(&k) {
594                transport.add_known_secret(&s);
595            }
596            headers.push((k, s));
597        }
598        let path = match u.path() {
599            "" | "/" => "/gelf".to_owned(),
600            p => p.to_owned(),
601        };
602        let path = match u.query() {
603            Some(q) => format!("{path}?{q}"),
604            None => path,
605        };
606        Ok(GelfHttpTransport {
607            path,
608            transport,
609            gzip: opts.gzip,
610            bulk: opts.bulk,
611            headers,
612        })
613    }
614}
615
616#[async_trait]
617impl GelfTransport for GelfHttpTransport {
618    fn protocol(&self) -> GelfProtocol {
619        GelfProtocol::Http
620    }
621
622    /// Posts messages: one body of newline-terminated messages per call (bulk), or one request
623    /// per message without bulk. Every message ends with `\n`, also single ones (Graylog 7.0
624    /// drops the first message of the third and later request on a reused connection
625    /// otherwise). Bodies are gzip-compressed unless disabled.
626    async fn send(
627        &self,
628        messages: &[GelfMessage],
629        opts: GelfSendOptions,
630    ) -> Result<GelfSendResult> {
631        if self.transport.is_closed() {
632            return Err(closed_error("gelf.http"));
633        }
634        let gzip = opts.gzip.unwrap_or(self.gzip);
635        let groups: Vec<&[GelfMessage]> = if self.bulk {
636            if messages.is_empty() {
637                Vec::new()
638            } else {
639                vec![messages]
640            }
641        } else {
642            messages.chunks(1).collect()
643        };
644        let mut res = GelfSendResult::default();
645        for g in groups {
646            let mut body = Vec::new();
647            for m in g {
648                body.extend(m.encode());
649                body.push(b'\n');
650            }
651            let mut req = self
652                .transport
653                .post(&self.path)
654                .operation("gelf.http")
655                .header(HEADER_REQUESTED_BY, DEFAULT_REQUESTED_BY);
656            for (k, v) in &self.headers {
657                req = if is_sensitive_header(k) {
658                    req.secret_header(k.clone(), v)
659                } else {
660                    req.header(k.clone(), v.expose_secret())
661                };
662            }
663            req = if gzip {
664                req.header("content-encoding", "gzip")
665                    .body(compress(GelfCompression::Gzip, &body), "application/json")
666            } else {
667                req.body(body, "application/json")
668            };
669            req.send().await?;
670            res.sent += g.len();
671        }
672        Ok(res)
673    }
674
675    async fn close(&self) -> Result<()> {
676        self.transport.close();
677        Ok(())
678    }
679}