Skip to main content

grow_graylog/
admin.rs

1//! Admin REST functions: inputs, streams and index sets.
2
3use std::collections::BTreeMap;
4use std::time::Duration;
5
6use serde::Deserialize;
7use serde_json::{json, Map, Value};
8
9use crate::client::{decode_json, require_id, seg, GraylogClient};
10use crate::config::GelfProtocol;
11use crate::core::redact::truncate;
12use crate::core::transport::MAX_VENDOR_MESSAGE;
13use crate::core::{poll_until, Error, ErrorCode, Method, PollOptions, Result};
14use crate::logging::SortOrder;
15use crate::models::{
16    IndexSet, Input, InputState, Stream, StreamPage, StreamRule, INPUT_STATE_FAILED,
17    INPUT_STATE_RUNNING, INPUT_TYPE_GELF_HTTP, INPUT_TYPE_GELF_TCP, INPUT_TYPE_GELF_UDP,
18};
19
20const PATH_INPUTS: &str = "/api/system/inputs";
21const PATH_INPUT_TYPES: &str = "/api/system/inputs/types";
22const PATH_INPUT_STATES: &str = "/api/system/inputstates";
23const PATH_STREAMS: &str = "/api/streams";
24const PATH_STREAMS_PAGE: &str = "/api/streams/paginated";
25const PATH_INDEX_SETS: &str = "/api/system/indices/index_sets";
26
27/// Options of [`GraylogClient::create_input`].
28#[derive(Debug, Clone, Default, PartialEq, Eq)]
29pub struct CreateInputOptions {
30    /// Title (default `"<class short name> (grow-codes)"`).
31    pub title: Option<String>,
32    /// Starts the input on one node only; `None` means a global input.
33    pub node: Option<String>,
34}
35
36/// Options of [`GraylogClient::ensure_gelf_input`]. The default is valid.
37#[derive(Debug, Clone, Default, PartialEq)]
38pub struct EnsureGelfInputOptions {
39    /// Title of a newly created input (default `"grow-codes GELF <PROTOCOL> <port>"`).
40    pub title: Option<String>,
41    /// Bind address of a new input (default `0.0.0.0`); an existing input matches when its bind
42    /// address equals this value (or is empty).
43    pub bind_address: Option<String>,
44    /// Merged over the default configuration of a new input (for example
45    /// `{"tls_enable": true}` or `{"enable_bulk_receiving": false}`).
46    pub configuration: Map<String, Value>,
47    /// Bound on the wait for `RUNNING` (default 30 s).
48    pub timeout: Option<Duration>,
49    /// Pause between state polls (default 500 ms).
50    pub interval: Option<Duration>,
51}
52
53/// Result of [`GraylogClient::ensure_gelf_input`].
54#[derive(Debug, Clone, Default, PartialEq)]
55pub struct EnsuredInput {
56    /// The running input.
57    pub input: Input,
58    /// Whether this call created it.
59    pub created: bool,
60}
61
62/// Options of [`GraylogClient::list_streams`]. The default is page 1, 50 per page, sorted by
63/// title ascending.
64#[derive(Debug, Clone, Default, PartialEq, Eq)]
65pub struct ListStreamsOptions {
66    /// 1-based page (default 1).
67    pub page: Option<u32>,
68    /// Page size (default 50).
69    pub per_page: Option<u32>,
70    /// Filter in Graylog's stream search syntax (for example a title).
71    pub query: Option<String>,
72    /// Sort field (default `title`).
73    pub sort: Option<String>,
74    /// Sort order (default ascending).
75    pub order: Option<SortOrder>,
76}
77
78/// Description of a new stream ([`GraylogClient::create_stream`]). `title` is required.
79#[derive(Debug, Clone, Default, PartialEq)]
80pub struct CreateStreamOptions {
81    /// Title.
82    pub title: String,
83    /// Description.
84    pub description: String,
85    /// Target index set (default: the default index set, else the first one).
86    pub index_set_id: Option<String>,
87    /// Routing rules.
88    pub rules: Vec<StreamRule>,
89    /// `AND` (default: every rule must match) or `OR`.
90    pub matching_type: Option<String>,
91    /// Whether matches leave the default stream.
92    pub remove_matches_from_default_stream: bool,
93    /// Keeps the new stream paused (Graylog creates streams paused; `create_stream` resumes them
94    /// unless this is set).
95    pub disabled: bool,
96}
97
98impl CreateStreamOptions {
99    /// Options with a title.
100    pub fn new(title: impl Into<String>) -> Self {
101        CreateStreamOptions {
102            title: title.into(),
103            ..CreateStreamOptions::default()
104        }
105    }
106    /// Sets the description.
107    pub fn description(mut self, description: impl Into<String>) -> Self {
108        self.description = description.into();
109        self
110    }
111    /// Adds a rule.
112    pub fn rule(mut self, rule: StreamRule) -> Self {
113        self.rules.push(rule);
114        self
115    }
116}
117
118/// Default configuration of a GELF input per protocol (the same as the Docker bootstrap).
119pub(crate) fn gelf_input_defaults(
120    protocol: GelfProtocol,
121) -> Result<(&'static str, Map<String, Value>)> {
122    let v = match protocol {
123        GelfProtocol::Udp => (
124            INPUT_TYPE_GELF_UDP,
125            json!({"recv_buffer_size": 262144, "number_worker_threads": 2, "decompress_size_limit": 8388608}),
126        ),
127        GelfProtocol::Tcp => (
128            INPUT_TYPE_GELF_TCP,
129            json!({"recv_buffer_size": 1048576, "number_worker_threads": 2, "tls_enable": false,
130                "tcp_keepalive": false, "use_null_delimiter": true, "max_message_size": 2097152,
131                "decompress_size_limit": 8388608}),
132        ),
133        GelfProtocol::Http => (
134            INPUT_TYPE_GELF_HTTP,
135            json!({"recv_buffer_size": 1048576, "number_worker_threads": 2, "tls_enable": false,
136                "tcp_keepalive": false, "enable_bulk_receiving": true, "enable_cors": true,
137                "max_chunk_size": 8388608, "idle_writer_timeout": 60, "decompress_size_limit": 8388608}),
138        ),
139        GelfProtocol::Otlp => {
140            return Err(Error::validation(
141                "ensureGelfInput: protocol must be udp, tcp or http",
142            ))
143        }
144    };
145    let Value::Object(map) = v.1 else {
146        unreachable!("literal objects")
147    };
148    Ok((v.0, map))
149}
150
151impl GraylogClient {
152    /// Every input (`GET /api/system/inputs`).
153    pub async fn list_inputs(&self) -> Result<Vec<Input>> {
154        #[derive(Deserialize)]
155        struct Body {
156            #[serde(default)]
157            inputs: Option<Vec<Input>>,
158        }
159        let body: Body = self.get_json(PATH_INPUTS, "listInputs").await?;
160        Ok(body.inputs.unwrap_or_default())
161    }
162
163    /// One input; `NOT_FOUND` when it does not exist.
164    pub async fn get_input(&self, id: &str) -> Result<Input> {
165        require_id("getInput", "id", id)?;
166        self.get_json(
167            &format!("{PATH_INPUTS}/{}", seg(id)),
168            &format!("getInput {id}"),
169        )
170        .await
171    }
172
173    /// Installed input types (class → display name).
174    pub async fn list_input_types(&self) -> Result<BTreeMap<String, String>> {
175        #[derive(Deserialize)]
176        struct Body {
177            #[serde(default)]
178            types: Option<BTreeMap<String, Value>>,
179        }
180        let body: Body = self.get_json(PATH_INPUT_TYPES, "listInputTypes").await?;
181        Ok(body
182            .types
183            .unwrap_or_default()
184            .into_iter()
185            .map(|(k, v)| (k, v.as_str().unwrap_or_default().to_owned()))
186            .collect())
187    }
188
189    /// Runtime state of an input; `NOT_FOUND` when it is not running on any node.
190    pub async fn get_input_state(&self, id: &str) -> Result<InputState> {
191        require_id("getInputState", "id", id)?;
192        self.get_json(
193            &format!("{PATH_INPUT_STATES}/{}", seg(id)),
194            &format!("getInputState {id}"),
195        )
196        .await
197    }
198
199    /// Creates an input of class `input_type` with `configuration` (`POST /api/system/inputs`)
200    /// and returns its id. An unknown type is `NOT_FOUND` (Graylog answers 404). Graylog does
201    /// not reject a busy port at create time: the input is created and its state becomes
202    /// `FAILED` (see [`get_input_state`](Self::get_input_state) or
203    /// [`ensure_gelf_input`](Self::ensure_gelf_input)).
204    pub async fn create_input(
205        &self,
206        input_type: &str,
207        configuration: Map<String, Value>,
208        opts: CreateInputOptions,
209    ) -> Result<String> {
210        require_id("createInput", "type", input_type)?;
211        let title = opts.title.filter(|t| !t.is_empty()).unwrap_or_else(|| {
212            let short = input_type.rsplit('.').next().unwrap_or(input_type);
213            format!("{short} (grow-codes)")
214        });
215        let node = opts.node.filter(|n| !n.is_empty());
216        let mut body = json!({
217            "title": title, "type": input_type, "global": node.is_none(),
218            "configuration": configuration,
219        });
220        if let Some(n) = node {
221            body["node"] = json!(n);
222        }
223        let resp = self
224            .send(
225                self.request(Method::POST, PATH_INPUTS, "createInput")
226                    .json(&body),
227            )
228            .await?;
229        let out: Value = decode_json(&resp, "createInput")?;
230        match out.get("id").and_then(Value::as_str) {
231            Some(id) if !id.is_empty() => Ok(id.to_owned()),
232            _ => Err(Error::unavailable("createInput: response has no input id")),
233        }
234    }
235
236    /// Deletes an input (`DELETE /api/system/inputs/{id}`); `NOT_FOUND` when it does not exist.
237    pub async fn delete_input(&self, id: &str) -> Result<()> {
238        require_id("deleteInput", "id", id)?;
239        self.send(self.request(
240            Method::DELETE,
241            &format!("{PATH_INPUTS}/{}", seg(id)),
242            &format!("deleteInput {id}"),
243        ))
244        .await
245        .map(|_| ())
246    }
247
248    async fn find_gelf_input(
249        &self,
250        input_type: &str,
251        port: u16,
252        bind: &str,
253    ) -> Result<Option<Input>> {
254        Ok(self.list_inputs().await?.into_iter().find(|i| {
255            i.input_type == input_type
256                && i.port() == port
257                && (i.bind_address().is_empty() || i.bind_address() == bind)
258        }))
259    }
260
261    /// A running GELF input of `protocol` on `port`: an existing input of the same type, port
262    /// and bind address is reused, otherwise one is created. It then waits until the input is
263    /// `RUNNING`; a `FAILED` input (for example a port already used by another input type) is a
264    /// `CONFLICT` error carrying Graylog's message, `TIMEOUT` after `opts.timeout`. An input
265    /// created by this call is deleted again when it does not start.
266    pub async fn ensure_gelf_input(
267        &self,
268        protocol: GelfProtocol,
269        port: u16,
270        opts: EnsureGelfInputOptions,
271    ) -> Result<EnsuredInput> {
272        const OP: &str = "ensureGelfInput";
273        let (input_type, mut config) = gelf_input_defaults(protocol)?;
274        if port == 0 {
275            return Err(Error::validation(format!(
276                "{OP}: port must be an integer 1..65535"
277            )));
278        }
279        let bind = opts
280            .bind_address
281            .clone()
282            .filter(|b| !b.is_empty())
283            .unwrap_or_else(|| "0.0.0.0".to_owned());
284        let mut created = false;
285        let input = match self.find_gelf_input(input_type, port, &bind).await? {
286            Some(i) => i,
287            None => {
288                config.extend(opts.configuration.clone());
289                config.insert("bind_address".into(), json!(bind));
290                config.insert("port".into(), json!(port));
291                let title = opts
292                    .title
293                    .clone()
294                    .filter(|t| !t.is_empty())
295                    .unwrap_or_else(|| {
296                        format!(
297                            "grow-codes GELF {} {port}",
298                            protocol.as_str().to_ascii_uppercase()
299                        )
300                    });
301                let create = CreateInputOptions {
302                    title: Some(title),
303                    node: None,
304                };
305                match self.create_input(input_type, config, create).await {
306                    Ok(id) => {
307                        created = true;
308                        self.get_input(&id).await?
309                    }
310                    Err(err) => {
311                        // The create may have reached the server; re-list before failing.
312                        match self.find_gelf_input(input_type, port, &bind).await {
313                            Ok(Some(i)) => i,
314                            _ => return Err(err),
315                        }
316                    }
317                }
318            }
319        };
320        let poll = PollOptions {
321            timeout: opts.timeout.unwrap_or(Duration::from_secs(30)),
322            interval: opts.interval.unwrap_or(Duration::from_millis(500)),
323        };
324        let id = input.id.clone();
325        let waited = poll_until(OP, poll, || async {
326            match self.get_input_state(&id).await {
327                Err(e) if e.code() == ErrorCode::NotFound => Ok(None),
328                Err(e) => Err(e),
329                Ok(st) if st.state == INPUT_STATE_FAILED => {
330                    let mut msg = format!("{OP} {protocol}:{port}: input failed to start");
331                    if !st.detailed_message.is_empty() {
332                        msg.push_str(": ");
333                        msg.push_str(&truncate(
334                            &self.transport().redact(&st.detailed_message),
335                            MAX_VENDOR_MESSAGE,
336                        ));
337                    }
338                    Err(Error::conflict(msg))
339                }
340                Ok(st) => Ok((st.state == INPUT_STATE_RUNNING).then_some(())),
341            }
342        })
343        .await;
344        if let Err(err) = waited {
345            if created && err.code() != ErrorCode::Cancelled {
346                let _ = self.delete_input(&id).await;
347            }
348            return Err(err);
349        }
350        Ok(EnsuredInput { input, created })
351    }
352
353    /// One page of streams (`GET /api/streams/paginated`).
354    pub async fn list_streams(&self, opts: ListStreamsOptions) -> Result<StreamPage> {
355        #[derive(Deserialize, Default)]
356        struct Pagination {
357            #[serde(default)]
358            total: u64,
359            #[serde(default)]
360            page: u32,
361            #[serde(default)]
362            per_page: u32,
363        }
364        #[derive(Deserialize)]
365        struct Body {
366            #[serde(default)]
367            pagination: Option<Pagination>,
368            #[serde(default)]
369            elements: Option<Vec<Stream>>,
370        }
371        let order = match opts.order.unwrap_or(SortOrder::Asc) {
372            SortOrder::Asc => "asc",
373            SortOrder::Desc => "desc",
374        };
375        let mut req = self
376            .request(Method::GET, PATH_STREAMS_PAGE, "listStreams")
377            .query(
378                "page",
379                opts.page.filter(|p| *p > 0).unwrap_or(1).to_string(),
380            )
381            .query(
382                "per_page",
383                opts.per_page.filter(|p| *p > 0).unwrap_or(50).to_string(),
384            )
385            .query(
386                "sort",
387                opts.sort
388                    .filter(|s| !s.is_empty())
389                    .unwrap_or_else(|| "title".into()),
390            )
391            .query("order", order);
392        if let Some(q) = opts.query.filter(|q| !q.is_empty()) {
393            req = req.query("query", q);
394        }
395        let resp = self.send(req).await?;
396        let body: Body = decode_json(&resp, "listStreams")?;
397        let pagination = body.pagination.unwrap_or_default();
398        Ok(StreamPage {
399            streams: body.elements.unwrap_or_default(),
400            total: pagination.total,
401            page: pagination.page,
402            per_page: pagination.per_page,
403        })
404    }
405
406    /// Follows [`list_streams`](Self::list_streams) pages and returns every stream.
407    pub(crate) async fn list_all_streams(&self) -> Result<Vec<Stream>> {
408        let mut out = Vec::new();
409        let mut page = 1;
410        loop {
411            let p = self
412                .list_streams(ListStreamsOptions {
413                    page: Some(page),
414                    per_page: Some(100),
415                    ..ListStreamsOptions::default()
416                })
417                .await?;
418            let empty = p.streams.is_empty();
419            out.extend(p.streams);
420            if empty || out.len() as u64 >= p.total {
421                return Ok(out);
422            }
423            page += 1;
424        }
425    }
426
427    /// One stream; `NOT_FOUND` when it does not exist.
428    pub async fn get_stream(&self, id: &str) -> Result<Stream> {
429        require_id("getStream", "id", id)?;
430        self.get_json(
431            &format!("{PATH_STREAMS}/{}", seg(id)),
432            &format!("getStream {id}"),
433        )
434        .await
435    }
436
437    /// Creates a stream (`POST /api/streams`) and, unless `opts.disabled`, resumes it so it
438    /// starts routing messages. Returns the stream id.
439    pub async fn create_stream(&self, opts: CreateStreamOptions) -> Result<String> {
440        const OP: &str = "createStream";
441        if opts.title.trim().is_empty() {
442            return Err(Error::validation(format!("{OP}: title is required")));
443        }
444        let matching = opts
445            .matching_type
446            .as_deref()
447            .filter(|m| !m.is_empty())
448            .unwrap_or("AND")
449            .to_ascii_uppercase();
450        if matching != "AND" && matching != "OR" {
451            return Err(Error::validation(format!(
452                "{OP}: matchingType must be \"AND\" or \"OR\""
453            )));
454        }
455        let index_set_id = match opts.index_set_id.filter(|s| !s.is_empty()) {
456            Some(id) => id,
457            None => {
458                let sets = self.list_index_sets().await?;
459                sets.iter()
460                    .find(|s| s.default)
461                    .or(sets.first())
462                    .map(|s| s.id.clone())
463                    .ok_or_else(|| Error::validation(format!("{OP}: no index set available")))?
464            }
465        };
466        let entity = json!({
467            "title": opts.title, "description": opts.description, "index_set_id": index_set_id,
468            "matching_type": matching,
469            "remove_matches_from_default_stream": opts.remove_matches_from_default_stream,
470            "rules": opts.rules,
471        });
472        let resp = self
473            .send(
474                self.request(Method::POST, PATH_STREAMS, OP)
475                    .json(&json!({ "entity": entity })),
476            )
477            .await?;
478        let out: Value = decode_json(&resp, OP)?;
479        let id = match out.get("stream_id").and_then(Value::as_str) {
480            Some(id) if !id.is_empty() => id.to_owned(),
481            _ => {
482                return Err(Error::unavailable(format!(
483                    "{OP}: response has no stream id"
484                )))
485            }
486        };
487        if !opts.disabled {
488            self.resume_stream(&id).await?;
489        }
490        Ok(id)
491    }
492
493    /// Starts a paused stream (`POST /api/streams/{id}/resume`).
494    pub async fn resume_stream(&self, id: &str) -> Result<()> {
495        require_id("resumeStream", "id", id)?;
496        self.send(
497            self.request(
498                Method::POST,
499                &format!("{PATH_STREAMS}/{}/resume", seg(id)),
500                &format!("resumeStream {id}"),
501            )
502            .idempotent(true),
503        )
504        .await
505        .map(|_| ())
506    }
507
508    /// Deletes a stream (`DELETE /api/streams/{id}`); `NOT_FOUND` when it does not exist.
509    pub async fn delete_stream(&self, id: &str) -> Result<()> {
510        require_id("deleteStream", "id", id)?;
511        self.send(self.request(
512            Method::DELETE,
513            &format!("{PATH_STREAMS}/{}", seg(id)),
514            &format!("deleteStream {id}"),
515        ))
516        .await
517        .map(|_| ())
518    }
519
520    /// Every index set (`GET /api/system/indices/index_sets?skip_stats=true`).
521    pub async fn list_index_sets(&self) -> Result<Vec<IndexSet>> {
522        #[derive(Deserialize)]
523        struct Body {
524            #[serde(default)]
525            index_sets: Option<Vec<IndexSet>>,
526        }
527        let resp = self
528            .send(
529                self.request(Method::GET, PATH_INDEX_SETS, "listIndexSets")
530                    .query("skip_stats", "true"),
531            )
532            .await?;
533        let body: Body = decode_json(&resp, "listIndexSets")?;
534        Ok(body.index_sets.unwrap_or_default())
535    }
536}