Skip to main content

grow_graylog/
search.rs

1//! Search (Views API), stored messages and their normalization into [`LogRecord`]s.
2
3use std::collections::BTreeMap;
4use std::time::{Duration, SystemTime, UNIX_EPOCH};
5
6use serde_json::{json, Map, Value};
7
8use crate::client::{decode_json, require_id, seg, GraylogClient};
9use crate::core::redact::truncate;
10use crate::core::transport::MAX_VENDOR_MESSAGE;
11use crate::core::{Error, Method, Result};
12use crate::gelf::gelf_level_to_severity;
13use crate::logging::{Attributes, LogRecord, Severity, SortOrder};
14use crate::models::{format_graylog_time, parse_graylog_time};
15
16const PATH_SEARCH_SYNC: &str = "/api/views/search/sync";
17const PATH_MESSAGES: &str = "/api/messages";
18
19/// Sort field of [`GraylogClient::search`] when none is given.
20pub const DEFAULT_MESSAGE_SORT_FIELD: &str = "timestamp";
21/// Page size of [`GraylogClient::search`] when none is given.
22pub const DEFAULT_SEARCH_LIMIT: usize = 100;
23/// Largest page `search` requests (OpenSearch's default `max_result_window` for offset + limit).
24pub const MAX_SEARCH_LIMIT: usize = 10_000;
25/// Relative range of [`GraylogClient::search`] when no range is given (300 s).
26pub const DEFAULT_SEARCH_RANGE: Duration = Duration::from_secs(300);
27
28/// One sort key of [`GraylogClient::search`].
29#[derive(Debug, Clone, PartialEq, Eq)]
30pub struct SearchSort {
31    /// Field name.
32    pub field: String,
33    /// Order (default descending).
34    pub order: SortOrder,
35}
36
37impl SearchSort {
38    /// Ascending on `field`.
39    pub fn asc(field: impl Into<String>) -> Self {
40        SearchSort {
41            field: field.into(),
42            order: SortOrder::Asc,
43        }
44    }
45    /// Descending on `field`.
46    pub fn desc(field: impl Into<String>) -> Self {
47        SearchSort {
48            field: field.into(),
49            order: SortOrder::Desc,
50        }
51    }
52}
53
54/// Options of [`GraylogClient::search`]. Give at most one of `range`, `start`/`end` and
55/// `keyword`; the default searches the last 5 minutes of every readable stream, newest first,
56/// 100 per page.
57#[derive(Debug, Clone, Default, PartialEq, Eq)]
58pub struct MessageSearchOptions {
59    /// Relative range ending now (whole seconds, rounded up; default 300 s).
60    pub range: Option<Duration>,
61    /// Start of an absolute range (default `end` − 300 s).
62    pub start: Option<SystemTime>,
63    /// End of an absolute range (default now).
64    pub end: Option<SystemTime>,
65    /// Graylog keyword range such as `last five minutes`.
66    pub keyword: Option<String>,
67    /// Page size (default 100, at most 10 000).
68    pub limit: Option<usize>,
69    /// Matches to skip (pagination).
70    pub offset: usize,
71    /// Returned fields (Graylog always adds `_id`); empty = all.
72    pub fields: Vec<String>,
73    /// Sort keys (default `timestamp` descending); later keys break ties.
74    pub sort: Vec<SearchSort>,
75    /// Stream ids to search (default: every stream the caller may read).
76    pub streams: Vec<String>,
77}
78
79/// One search hit or stored message.
80#[derive(Debug, Clone, Default, PartialEq)]
81pub struct Message {
82    /// Message id (`_id`); with `index` it identifies the message for
83    /// [`GraylogClient::get_message`].
84    pub id: String,
85    /// Index that stores the message.
86    pub index: String,
87    /// Message timestamp.
88    pub timestamp: Option<SystemTime>,
89    /// Every returned field as JSON.
90    pub fields: Map<String, Value>,
91}
92
93impl Message {
94    fn new(fields: Map<String, Value>, index: String) -> Message {
95        let id = fields
96            .get("_id")
97            .and_then(Value::as_str)
98            .unwrap_or_default()
99            .to_owned();
100        let timestamp = fields
101            .get("timestamp")
102            .and_then(Value::as_str)
103            .and_then(parse_graylog_time);
104        Message {
105            id,
106            index,
107            timestamp,
108            fields,
109        }
110    }
111
112    /// The message normalized into a [`LogRecord`] (see [`log_record_from_message`]).
113    pub fn record(&self) -> LogRecord {
114        log_record_from_message(&self.fields)
115    }
116}
117
118/// One page of [`GraylogClient::search`].
119#[derive(Debug, Clone, Default, PartialEq)]
120pub struct MessageSearchResult {
121    /// Hits of this page.
122    pub messages: Vec<Message>,
123    /// Matches of the whole query.
124    pub total: u64,
125    /// Effective start of the searched range.
126    pub from: Option<SystemTime>,
127    /// Effective end of the searched range.
128    pub to: Option<SystemTime>,
129    /// Offset of the next page (`None` when there is none).
130    pub next_offset: Option<usize>,
131}
132
133impl MessageSearchResult {
134    /// Every hit normalized into a [`LogRecord`].
135    pub fn records(&self) -> Vec<LogRecord> {
136        self.messages.iter().map(Message::record).collect()
137    }
138}
139
140fn order_str(o: SortOrder) -> &'static str {
141    match o {
142        SortOrder::Asc => "ASC",
143        SortOrder::Desc => "DESC",
144    }
145}
146
147fn timerange(op: &str, opts: &MessageSearchOptions) -> Result<Value> {
148    let kinds = [
149        opts.range.is_some(),
150        opts.start.is_some() || opts.end.is_some(),
151        opts.keyword.is_some(),
152    ]
153    .iter()
154    .filter(|b| **b)
155    .count();
156    if kinds > 1 {
157        return Err(Error::validation(format!(
158            "{op}: give only one of range, start/end and keyword"
159        )));
160    }
161    if opts.start.is_some() || opts.end.is_some() {
162        let end = opts.end.unwrap_or_else(SystemTime::now);
163        let start = opts
164            .start
165            .unwrap_or_else(|| end.checked_sub(DEFAULT_SEARCH_RANGE).unwrap_or(UNIX_EPOCH));
166        if start > end {
167            return Err(Error::validation(format!("{op}: start is after end")));
168        }
169        return Ok(
170            json!({"type": "absolute", "from": format_graylog_time(start), "to": format_graylog_time(end)}),
171        );
172    }
173    if let Some(k) = &opts.keyword {
174        if k.trim().is_empty() {
175            return Err(Error::validation(format!(
176                "{op}: keyword must not be empty"
177            )));
178        }
179        return Ok(json!({"type": "keyword", "keyword": k}));
180    }
181    let range = opts.range.unwrap_or(DEFAULT_SEARCH_RANGE);
182    if range.is_zero() {
183        return Err(Error::validation(format!("{op}: range must be positive")));
184    }
185    Ok(json!({"type": "relative", "range": range.as_secs_f64().ceil() as u64}))
186}
187
188fn first_error(body: &Value) -> Option<String> {
189    let errs = body
190        .get("errors")
191        .and_then(Value::as_array)
192        .into_iter()
193        .flatten()
194        .chain(
195            body.pointer("/results/q1/errors")
196                .and_then(Value::as_array)
197                .into_iter()
198                .flatten(),
199        );
200    let mut any = false;
201    for e in errs {
202        any = true;
203        if let Some(d) = e.get("description").and_then(Value::as_str) {
204            if !d.is_empty() {
205                return Some(d.to_owned());
206            }
207        }
208    }
209    let exceptional = body
210        .pointer("/execution/completed_exceptionally")
211        .and_then(Value::as_bool)
212        .unwrap_or(false);
213    (any || exceptional).then(|| "query failed".to_owned())
214}
215
216impl GraylogClient {
217    /// Runs a Lucene query through the Views search API (`POST /api/views/search/sync`) and
218    /// returns one page of messages. An empty query matches everything. Query errors (Graylog
219    /// answers HTTP 200 with `errors[]`) are `VALIDATION` errors.
220    pub async fn search(
221        &self,
222        query: &str,
223        opts: MessageSearchOptions,
224    ) -> Result<MessageSearchResult> {
225        const OP: &str = "search";
226        let limit = opts.limit.unwrap_or(DEFAULT_SEARCH_LIMIT);
227        if !(1..=MAX_SEARCH_LIMIT).contains(&limit) {
228            return Err(Error::validation(format!(
229                "{OP}: limit must be an integer 1..10000"
230            )));
231        }
232        let sorts = if opts.sort.is_empty() {
233            vec![SearchSort::desc(DEFAULT_MESSAGE_SORT_FIELD)]
234        } else {
235            opts.sort.clone()
236        };
237        let mut sort_body = Vec::with_capacity(sorts.len());
238        for s in &sorts {
239            if s.field.trim().is_empty() {
240                return Err(Error::validation(format!(
241                    "{OP}: sort field must not be empty"
242                )));
243            }
244            sort_body.push(json!({"field": s.field, "order": order_str(s.order)}));
245        }
246        let query = if query.trim().is_empty() { "*" } else { query };
247        let tr = timerange(OP, &opts)?;
248        let mut st = json!({"id": "st1", "type": "messages", "limit": limit, "offset": opts.offset, "sort": sort_body});
249        if !opts.fields.is_empty() {
250            st["fields"] = json!(opts.fields);
251        }
252        let mut q = json!({
253            "id": "q1",
254            "query": {"type": "elasticsearch", "query_string": query},
255            "timerange": tr,
256            "search_types": [st],
257        });
258        if !opts.streams.is_empty() {
259            let filters: Vec<Value> = opts
260                .streams
261                .iter()
262                .map(|s| json!({"type": "stream", "id": s}))
263                .collect();
264            q["filter"] = json!({"type": "or", "filters": filters});
265        }
266        let resp = self
267            .send(
268                self.request(Method::POST, PATH_SEARCH_SYNC, OP)
269                    .json(&json!({ "queries": [q] }))
270                    .idempotent(true),
271            )
272            .await?;
273        let body: Value = decode_json(&resp, OP)?;
274        if let Some(text) = first_error(&body) {
275            return Err(Error::validation(format!(
276                "{OP}: {}",
277                truncate(&self.transport().redact(&text), MAX_VENDOR_MESSAGE)
278            )));
279        }
280        let st = body
281            .pointer("/results/q1/search_types/st1")
282            .cloned()
283            .unwrap_or(Value::Null);
284        let total = st.get("total_results").and_then(Value::as_u64).unwrap_or(0);
285        let time = |k: &str| {
286            st.pointer(&format!("/effective_timerange/{k}"))
287                .and_then(Value::as_str)
288                .and_then(parse_graylog_time)
289        };
290        let messages: Vec<Message> = st
291            .get("messages")
292            .and_then(Value::as_array)
293            .into_iter()
294            .flatten()
295            .map(|h| {
296                let fields = h
297                    .get("message")
298                    .and_then(Value::as_object)
299                    .cloned()
300                    .unwrap_or_default();
301                let index = h
302                    .get("index")
303                    .and_then(Value::as_str)
304                    .unwrap_or_default()
305                    .to_owned();
306                Message::new(fields, index)
307            })
308            .collect();
309        let next = opts.offset + messages.len();
310        Ok(MessageSearchResult {
311            next_offset: (!messages.is_empty() && (next as u64) < total).then_some(next),
312            from: time("from"),
313            to: time("to"),
314            total,
315            messages,
316        })
317    }
318
319    /// [`search`](Self::search) over the last `range` (relative, whole seconds rounded up).
320    /// `start`, `end` and `keyword` of `opts` must be unset.
321    pub async fn search_relative(
322        &self,
323        query: &str,
324        range: Duration,
325        mut opts: MessageSearchOptions,
326    ) -> Result<MessageSearchResult> {
327        if range.is_zero() {
328            return Err(Error::validation("searchRelative: range must be positive"));
329        }
330        opts.range = Some(range);
331        self.search(query, opts).await
332    }
333
334    /// One stored message by index and id (`GET /api/messages/{index}/{id}`); `NOT_FOUND` when
335    /// it does not exist or the caller may not read it. Search hits carry both values.
336    pub async fn get_message(&self, index: &str, id: &str) -> Result<Message> {
337        const OP: &str = "getMessage";
338        require_id(OP, "index", index)?;
339        require_id(OP, "id", id)?;
340        let body: Value = self
341            .get_json(
342                &format!("{PATH_MESSAGES}/{}/{}", seg(index), seg(id)),
343                &format!("{OP} {index}/{id}"),
344            )
345            .await?;
346        let fields = body
347            .pointer("/message/fields")
348            .and_then(Value::as_object)
349            .cloned()
350            .unwrap_or_default();
351        let idx = body
352            .get("index")
353            .and_then(Value::as_str)
354            .unwrap_or(index)
355            .to_owned();
356        let mut m = Message::new(fields, idx);
357        if m.id.is_empty() {
358            m.id = body
359                .pointer("/message/id")
360                .and_then(Value::as_str)
361                .unwrap_or_default()
362                .to_owned();
363        }
364        Ok(m)
365    }
366}
367
368/// Stored field names mapped back to OpenTelemetry resource keys (Graylog replaces `.` with
369/// `_`; OTLP resource attributes arrive as `otel_resource_attributes_<name>`).
370pub const RESOURCE_FIELDS: &[(&str, &str)] = &[
371    ("service_name", "service.name"),
372    ("service_version", "service.version"),
373    ("service_namespace", "service.namespace"),
374    ("service_instance_id", "service.instance.id"),
375    ("deployment_environment", "deployment.environment"),
376    ("deployment_environment_name", "deployment.environment.name"),
377];
378
379fn resource_key(field: &str) -> Option<&'static str> {
380    RESOURCE_FIELDS
381        .iter()
382        .find(|(k, _)| *k == field)
383        .map(|(_, v)| *v)
384}
385
386const SKIPPED_FIELDS: &[&str] = &[
387    "_id",
388    "message",
389    "timestamp",
390    "level",
391    "source",
392    "streams",
393    "trace_id",
394    "span_id",
395    "trace_flags",
396    "otel_trace_id",
397    "otel_span_id",
398    "otel_trace_flags",
399    "otel_severity_number",
400    "otel_severity_text",
401    "otel_time_unix_nano",
402    "otel_observed_time_unix_nano",
403    "otel_resource_schema_url",
404    "otel_scope_schema_url",
405];
406
407fn number_field(v: Option<&Value>) -> Option<f64> {
408    match v? {
409        Value::Number(n) => n.as_f64(),
410        Value::String(s) => s.trim().parse().ok(),
411        _ => None,
412    }
413}
414
415fn hex_id(len: usize, values: &[Option<&Value>]) -> Option<String> {
416    values.iter().flatten().find_map(|v| {
417        let s = v.as_str()?.trim().to_ascii_lowercase();
418        (s.len() == len && s.bytes().all(|b| b.is_ascii_hexdigit())).then_some(s)
419    })
420}
421
422fn string_field(v: Option<&Value>) -> String {
423    match v {
424        None | Some(Value::Null) => String::new(),
425        Some(Value::String(s)) => s.clone(),
426        Some(other) => other.to_string(),
427    }
428}
429
430/// Normalizes stored Graylog message fields (a search hit or [`GraylogClient::get_message`])
431/// into a [`LogRecord`]. OTLP fields (`otel_*`) win over GELF fields:
432/// - body = `message`, timestamp = `timestamp`, `resource["host.name"]` = `source`;
433/// - severity from `otel_severity_number` (named level of its range), else from the syslog
434///   `level` (0–2 FATAL, 3 ERROR, 4 WARN, 5–6 INFO, 7 DEBUG), else INFO;
435/// - `otel_resource_attributes_<k>` → `resource[RESOURCE_FIELDS[k] or k]`, stored fields named in
436///   [`RESOURCE_FIELDS`] → resource (unless already set), `otel_attributes_<k>` →
437///   `attributes[k]`, `attr_id` → `attributes["id"]`, every other field except `gl2_*` and the
438///   reserved ones → attributes;
439/// - trace id from `otel_trace_id` or `trace_id` (32 lower-hex), and only then span id (16
440///   lower-hex) and trace flags.
441pub fn log_record_from_message(fields: &Map<String, Value>) -> LogRecord {
442    let severity = match number_field(fields.get("otel_severity_number")) {
443        Some(n) if (1.0..=24.0).contains(&n) => Severity::from_number(n as u8),
444        _ => number_field(fields.get("level"))
445            .map(|l| gelf_level_to_severity(l as i64))
446            .unwrap_or(Severity::Info),
447    };
448    let mut resource = Attributes::new();
449    let mut attributes = Attributes::new();
450    if let Some(src) = fields
451        .get("source")
452        .and_then(Value::as_str)
453        .filter(|s| !s.is_empty())
454    {
455        resource.insert("host.name".into(), Value::String(src.to_owned()));
456    }
457    let mut late_resource: BTreeMap<String, Value> = BTreeMap::new();
458    for (k, v) in fields {
459        if v.is_null() || SKIPPED_FIELDS.contains(&k.as_str()) || k.starts_with("gl2_") {
460            continue;
461        }
462        if let Some(name) = k.strip_prefix("otel_resource_attributes_") {
463            let key = resource_key(name).unwrap_or(name);
464            resource.insert(key.to_owned(), v.clone());
465        } else if let Some(name) = k.strip_prefix("otel_attributes_") {
466            attributes.insert(name.to_owned(), v.clone());
467        } else if let Some(rk) = resource_key(k) {
468            late_resource.insert(rk.to_owned(), v.clone());
469        } else if k == "attr_id" {
470            attributes.insert("id".into(), v.clone());
471        } else {
472            attributes.insert(k.clone(), v.clone());
473        }
474    }
475    for (k, v) in late_resource {
476        resource.entry(k).or_insert(v);
477    }
478    let trace_id = hex_id(32, &[fields.get("otel_trace_id"), fields.get("trace_id")]);
479    let (span_id, trace_flags) = if trace_id.is_some() {
480        let span = hex_id(16, &[fields.get("otel_span_id"), fields.get("span_id")]);
481        let flags = [fields.get("otel_trace_flags"), fields.get("trace_flags")]
482            .into_iter()
483            .find_map(|f| number_field(f).filter(|n| (0.0..=255.0).contains(n)))
484            .map(|n| n as u8);
485        (span, flags)
486    } else {
487        (None, None)
488    };
489    LogRecord {
490        timestamp: fields
491            .get("timestamp")
492            .and_then(Value::as_str)
493            .and_then(parse_graylog_time)
494            .unwrap_or(UNIX_EPOCH),
495        observed_timestamp: None,
496        severity,
497        severity_text: severity.text().to_owned(),
498        body: string_field(fields.get("message")),
499        attributes,
500        resource,
501        trace_id,
502        span_id,
503        trace_flags,
504    }
505}