1use std::collections::BTreeMap;
4use std::time::{Duration, SystemTime, UNIX_EPOCH};
5
6use serde_json::{json, Map, Value};
7
8use crate::client::{decode_json, require_id, seg, GraylogClient};
9use crate::core::redact::truncate;
10use crate::core::transport::MAX_VENDOR_MESSAGE;
11use crate::core::{Error, Method, Result};
12use crate::gelf::gelf_level_to_severity;
13use crate::logging::{Attributes, LogRecord, Severity, SortOrder};
14use crate::models::{format_graylog_time, parse_graylog_time};
15
16const PATH_SEARCH_SYNC: &str = "/api/views/search/sync";
17const PATH_MESSAGES: &str = "/api/messages";
18
19pub const DEFAULT_MESSAGE_SORT_FIELD: &str = "timestamp";
21pub const DEFAULT_SEARCH_LIMIT: usize = 100;
23pub const MAX_SEARCH_LIMIT: usize = 10_000;
25pub const DEFAULT_SEARCH_RANGE: Duration = Duration::from_secs(300);
27
28#[derive(Debug, Clone, PartialEq, Eq)]
30pub struct SearchSort {
31 pub field: String,
33 pub order: SortOrder,
35}
36
37impl SearchSort {
38 pub fn asc(field: impl Into<String>) -> Self {
40 SearchSort {
41 field: field.into(),
42 order: SortOrder::Asc,
43 }
44 }
45 pub fn desc(field: impl Into<String>) -> Self {
47 SearchSort {
48 field: field.into(),
49 order: SortOrder::Desc,
50 }
51 }
52}
53
54#[derive(Debug, Clone, Default, PartialEq, Eq)]
58pub struct MessageSearchOptions {
59 pub range: Option<Duration>,
61 pub start: Option<SystemTime>,
63 pub end: Option<SystemTime>,
65 pub keyword: Option<String>,
67 pub limit: Option<usize>,
69 pub offset: usize,
71 pub fields: Vec<String>,
73 pub sort: Vec<SearchSort>,
75 pub streams: Vec<String>,
77}
78
79#[derive(Debug, Clone, Default, PartialEq)]
81pub struct Message {
82 pub id: String,
85 pub index: String,
87 pub timestamp: Option<SystemTime>,
89 pub fields: Map<String, Value>,
91}
92
93impl Message {
94 fn new(fields: Map<String, Value>, index: String) -> Message {
95 let id = fields
96 .get("_id")
97 .and_then(Value::as_str)
98 .unwrap_or_default()
99 .to_owned();
100 let timestamp = fields
101 .get("timestamp")
102 .and_then(Value::as_str)
103 .and_then(parse_graylog_time);
104 Message {
105 id,
106 index,
107 timestamp,
108 fields,
109 }
110 }
111
112 pub fn record(&self) -> LogRecord {
114 log_record_from_message(&self.fields)
115 }
116}
117
118#[derive(Debug, Clone, Default, PartialEq)]
120pub struct MessageSearchResult {
121 pub messages: Vec<Message>,
123 pub total: u64,
125 pub from: Option<SystemTime>,
127 pub to: Option<SystemTime>,
129 pub next_offset: Option<usize>,
131}
132
133impl MessageSearchResult {
134 pub fn records(&self) -> Vec<LogRecord> {
136 self.messages.iter().map(Message::record).collect()
137 }
138}
139
140fn order_str(o: SortOrder) -> &'static str {
141 match o {
142 SortOrder::Asc => "ASC",
143 SortOrder::Desc => "DESC",
144 }
145}
146
147fn timerange(op: &str, opts: &MessageSearchOptions) -> Result<Value> {
148 let kinds = [
149 opts.range.is_some(),
150 opts.start.is_some() || opts.end.is_some(),
151 opts.keyword.is_some(),
152 ]
153 .iter()
154 .filter(|b| **b)
155 .count();
156 if kinds > 1 {
157 return Err(Error::validation(format!(
158 "{op}: give only one of range, start/end and keyword"
159 )));
160 }
161 if opts.start.is_some() || opts.end.is_some() {
162 let end = opts.end.unwrap_or_else(SystemTime::now);
163 let start = opts
164 .start
165 .unwrap_or_else(|| end.checked_sub(DEFAULT_SEARCH_RANGE).unwrap_or(UNIX_EPOCH));
166 if start > end {
167 return Err(Error::validation(format!("{op}: start is after end")));
168 }
169 return Ok(
170 json!({"type": "absolute", "from": format_graylog_time(start), "to": format_graylog_time(end)}),
171 );
172 }
173 if let Some(k) = &opts.keyword {
174 if k.trim().is_empty() {
175 return Err(Error::validation(format!(
176 "{op}: keyword must not be empty"
177 )));
178 }
179 return Ok(json!({"type": "keyword", "keyword": k}));
180 }
181 let range = opts.range.unwrap_or(DEFAULT_SEARCH_RANGE);
182 if range.is_zero() {
183 return Err(Error::validation(format!("{op}: range must be positive")));
184 }
185 Ok(json!({"type": "relative", "range": range.as_secs_f64().ceil() as u64}))
186}
187
188fn first_error(body: &Value) -> Option<String> {
189 let errs = body
190 .get("errors")
191 .and_then(Value::as_array)
192 .into_iter()
193 .flatten()
194 .chain(
195 body.pointer("/results/q1/errors")
196 .and_then(Value::as_array)
197 .into_iter()
198 .flatten(),
199 );
200 let mut any = false;
201 for e in errs {
202 any = true;
203 if let Some(d) = e.get("description").and_then(Value::as_str) {
204 if !d.is_empty() {
205 return Some(d.to_owned());
206 }
207 }
208 }
209 let exceptional = body
210 .pointer("/execution/completed_exceptionally")
211 .and_then(Value::as_bool)
212 .unwrap_or(false);
213 (any || exceptional).then(|| "query failed".to_owned())
214}
215
216impl GraylogClient {
217 pub async fn search(
221 &self,
222 query: &str,
223 opts: MessageSearchOptions,
224 ) -> Result<MessageSearchResult> {
225 const OP: &str = "search";
226 let limit = opts.limit.unwrap_or(DEFAULT_SEARCH_LIMIT);
227 if !(1..=MAX_SEARCH_LIMIT).contains(&limit) {
228 return Err(Error::validation(format!(
229 "{OP}: limit must be an integer 1..10000"
230 )));
231 }
232 let sorts = if opts.sort.is_empty() {
233 vec![SearchSort::desc(DEFAULT_MESSAGE_SORT_FIELD)]
234 } else {
235 opts.sort.clone()
236 };
237 let mut sort_body = Vec::with_capacity(sorts.len());
238 for s in &sorts {
239 if s.field.trim().is_empty() {
240 return Err(Error::validation(format!(
241 "{OP}: sort field must not be empty"
242 )));
243 }
244 sort_body.push(json!({"field": s.field, "order": order_str(s.order)}));
245 }
246 let query = if query.trim().is_empty() { "*" } else { query };
247 let tr = timerange(OP, &opts)?;
248 let mut st = json!({"id": "st1", "type": "messages", "limit": limit, "offset": opts.offset, "sort": sort_body});
249 if !opts.fields.is_empty() {
250 st["fields"] = json!(opts.fields);
251 }
252 let mut q = json!({
253 "id": "q1",
254 "query": {"type": "elasticsearch", "query_string": query},
255 "timerange": tr,
256 "search_types": [st],
257 });
258 if !opts.streams.is_empty() {
259 let filters: Vec<Value> = opts
260 .streams
261 .iter()
262 .map(|s| json!({"type": "stream", "id": s}))
263 .collect();
264 q["filter"] = json!({"type": "or", "filters": filters});
265 }
266 let resp = self
267 .send(
268 self.request(Method::POST, PATH_SEARCH_SYNC, OP)
269 .json(&json!({ "queries": [q] }))
270 .idempotent(true),
271 )
272 .await?;
273 let body: Value = decode_json(&resp, OP)?;
274 if let Some(text) = first_error(&body) {
275 return Err(Error::validation(format!(
276 "{OP}: {}",
277 truncate(&self.transport().redact(&text), MAX_VENDOR_MESSAGE)
278 )));
279 }
280 let st = body
281 .pointer("/results/q1/search_types/st1")
282 .cloned()
283 .unwrap_or(Value::Null);
284 let total = st.get("total_results").and_then(Value::as_u64).unwrap_or(0);
285 let time = |k: &str| {
286 st.pointer(&format!("/effective_timerange/{k}"))
287 .and_then(Value::as_str)
288 .and_then(parse_graylog_time)
289 };
290 let messages: Vec<Message> = st
291 .get("messages")
292 .and_then(Value::as_array)
293 .into_iter()
294 .flatten()
295 .map(|h| {
296 let fields = h
297 .get("message")
298 .and_then(Value::as_object)
299 .cloned()
300 .unwrap_or_default();
301 let index = h
302 .get("index")
303 .and_then(Value::as_str)
304 .unwrap_or_default()
305 .to_owned();
306 Message::new(fields, index)
307 })
308 .collect();
309 let next = opts.offset + messages.len();
310 Ok(MessageSearchResult {
311 next_offset: (!messages.is_empty() && (next as u64) < total).then_some(next),
312 from: time("from"),
313 to: time("to"),
314 total,
315 messages,
316 })
317 }
318
319 pub async fn search_relative(
322 &self,
323 query: &str,
324 range: Duration,
325 mut opts: MessageSearchOptions,
326 ) -> Result<MessageSearchResult> {
327 if range.is_zero() {
328 return Err(Error::validation("searchRelative: range must be positive"));
329 }
330 opts.range = Some(range);
331 self.search(query, opts).await
332 }
333
334 pub async fn get_message(&self, index: &str, id: &str) -> Result<Message> {
337 const OP: &str = "getMessage";
338 require_id(OP, "index", index)?;
339 require_id(OP, "id", id)?;
340 let body: Value = self
341 .get_json(
342 &format!("{PATH_MESSAGES}/{}/{}", seg(index), seg(id)),
343 &format!("{OP} {index}/{id}"),
344 )
345 .await?;
346 let fields = body
347 .pointer("/message/fields")
348 .and_then(Value::as_object)
349 .cloned()
350 .unwrap_or_default();
351 let idx = body
352 .get("index")
353 .and_then(Value::as_str)
354 .unwrap_or(index)
355 .to_owned();
356 let mut m = Message::new(fields, idx);
357 if m.id.is_empty() {
358 m.id = body
359 .pointer("/message/id")
360 .and_then(Value::as_str)
361 .unwrap_or_default()
362 .to_owned();
363 }
364 Ok(m)
365 }
366}
367
368pub const RESOURCE_FIELDS: &[(&str, &str)] = &[
371 ("service_name", "service.name"),
372 ("service_version", "service.version"),
373 ("service_namespace", "service.namespace"),
374 ("service_instance_id", "service.instance.id"),
375 ("deployment_environment", "deployment.environment"),
376 ("deployment_environment_name", "deployment.environment.name"),
377];
378
379fn resource_key(field: &str) -> Option<&'static str> {
380 RESOURCE_FIELDS
381 .iter()
382 .find(|(k, _)| *k == field)
383 .map(|(_, v)| *v)
384}
385
386const SKIPPED_FIELDS: &[&str] = &[
387 "_id",
388 "message",
389 "timestamp",
390 "level",
391 "source",
392 "streams",
393 "trace_id",
394 "span_id",
395 "trace_flags",
396 "otel_trace_id",
397 "otel_span_id",
398 "otel_trace_flags",
399 "otel_severity_number",
400 "otel_severity_text",
401 "otel_time_unix_nano",
402 "otel_observed_time_unix_nano",
403 "otel_resource_schema_url",
404 "otel_scope_schema_url",
405];
406
407fn number_field(v: Option<&Value>) -> Option<f64> {
408 match v? {
409 Value::Number(n) => n.as_f64(),
410 Value::String(s) => s.trim().parse().ok(),
411 _ => None,
412 }
413}
414
415fn hex_id(len: usize, values: &[Option<&Value>]) -> Option<String> {
416 values.iter().flatten().find_map(|v| {
417 let s = v.as_str()?.trim().to_ascii_lowercase();
418 (s.len() == len && s.bytes().all(|b| b.is_ascii_hexdigit())).then_some(s)
419 })
420}
421
422fn string_field(v: Option<&Value>) -> String {
423 match v {
424 None | Some(Value::Null) => String::new(),
425 Some(Value::String(s)) => s.clone(),
426 Some(other) => other.to_string(),
427 }
428}
429
430pub fn log_record_from_message(fields: &Map<String, Value>) -> LogRecord {
442 let severity = match number_field(fields.get("otel_severity_number")) {
443 Some(n) if (1.0..=24.0).contains(&n) => Severity::from_number(n as u8),
444 _ => number_field(fields.get("level"))
445 .map(|l| gelf_level_to_severity(l as i64))
446 .unwrap_or(Severity::Info),
447 };
448 let mut resource = Attributes::new();
449 let mut attributes = Attributes::new();
450 if let Some(src) = fields
451 .get("source")
452 .and_then(Value::as_str)
453 .filter(|s| !s.is_empty())
454 {
455 resource.insert("host.name".into(), Value::String(src.to_owned()));
456 }
457 let mut late_resource: BTreeMap<String, Value> = BTreeMap::new();
458 for (k, v) in fields {
459 if v.is_null() || SKIPPED_FIELDS.contains(&k.as_str()) || k.starts_with("gl2_") {
460 continue;
461 }
462 if let Some(name) = k.strip_prefix("otel_resource_attributes_") {
463 let key = resource_key(name).unwrap_or(name);
464 resource.insert(key.to_owned(), v.clone());
465 } else if let Some(name) = k.strip_prefix("otel_attributes_") {
466 attributes.insert(name.to_owned(), v.clone());
467 } else if let Some(rk) = resource_key(k) {
468 late_resource.insert(rk.to_owned(), v.clone());
469 } else if k == "attr_id" {
470 attributes.insert("id".into(), v.clone());
471 } else {
472 attributes.insert(k.clone(), v.clone());
473 }
474 }
475 for (k, v) in late_resource {
476 resource.entry(k).or_insert(v);
477 }
478 let trace_id = hex_id(32, &[fields.get("otel_trace_id"), fields.get("trace_id")]);
479 let (span_id, trace_flags) = if trace_id.is_some() {
480 let span = hex_id(16, &[fields.get("otel_span_id"), fields.get("span_id")]);
481 let flags = [fields.get("otel_trace_flags"), fields.get("trace_flags")]
482 .into_iter()
483 .find_map(|f| number_field(f).filter(|n| (0.0..=255.0).contains(n)))
484 .map(|n| n as u8);
485 (span, flags)
486 } else {
487 (None, None)
488 };
489 LogRecord {
490 timestamp: fields
491 .get("timestamp")
492 .and_then(Value::as_str)
493 .and_then(parse_graylog_time)
494 .unwrap_or(UNIX_EPOCH),
495 observed_timestamp: None,
496 severity,
497 severity_text: severity.text().to_owned(),
498 body: string_field(fields.get("message")),
499 attributes,
500 resource,
501 trace_id,
502 span_id,
503 trace_flags,
504 }
505}