loki_logger/
lib.rs

1#![deny(missing_docs)]
2#![deny(rustdoc::missing_doc_code_examples)]
3//! A [loki](https://grafana.com/oss/loki/) logger for the [`log`](https://crates.io/crates/log) facade.
4//! One event is written and send to loki per log call. Each event contain the time in nano second
5//! it was scheduled to be sent, in most cases, when the logging occured.
6//!
7//! # Examples
8//!
9//! You simply need to specify your [loki push URL](https://grafana.com/docs/loki/latest/api/#post-lokiapiv1push) and the minimum log level to start the logger.
10//!
11//! ```rust
12//! # extern crate log;
13//! # extern crate loki_logger;
14//! use log::LevelFilter;
15//!
16//! # #[tokio::main]
17//! # async fn main() {
18//! loki_logger::init(
19//!     "http://loki:3100/loki/api/v1/push",
20//!     log::LevelFilter::Info,
21//! ).unwrap();
22//!
23//! log::info!("Logged into Loki !");
24//! # }
25//! ```
26//!
27//! Or specify [static labels](https://grafana.com/docs/loki/latest/best-practices/#static-labels-are-good) to use in your loki streams.
28//! Those labels are overwriten by event-specific label, if any.
29//!
30//! ```rust
31//! # extern crate log;
32//! # extern crate loki_logger;
33//! # use std::iter::FromIterator;
34//! use std::collections::HashMap;
35//! use log::LevelFilter;
36//!
37//! # #[tokio::main]
38//! # async fn main() {
39//! let initial_labels = HashMap::from_iter([
40//!     ("application".to_string(), "loki_logger".to_string()),
41//!     ("environment".to_string(), "development".to_string()),
42//! ]);
43//!
44//! loki_logger::init_with_labels(
45//!     "http://loki:3100/loki/api/v1/push",
46//!     log::LevelFilter::Info,
47//!     initial_labels,
48//! ).unwrap();
49//!
50//! log::info!("Logged into Loki !");
51//! # }
52//! ```
53//! # Log format
54//!
55//! Each and every log event sent to loki will contain at least the [`level`](log::Level) of the event as well as the [time in nanoseconds](std::time::Duration::as_nanos) of the event scheduling.
56//!
57//! # Notice on extra labels
58//!
59//! Starting from 0.4.7, the [`log`](https://crates.io/crates/log) crate started introducing the new key/value system for structured logging.
60//!
61//! The loki_logger crate makes heavy use of such system as to create and send custom loki labels.
62//!
63//! If you want to use the key:value tag system, you have to use the git version of the log crate and enable the [`kv_unstable`](https://docs.rs/crate/log/0.4.14/features#kv_unstable) feature:
64//!
65//! ```toml
66//! [dependencies.log]
67//! # It is recommended that you pin this version to a specific commit to avoid issues.
68//! git = "https://github.com/rust-lang/log.git"
69//! branch = "kv_macro"
70//! features = ["kv_unstable"]
71//! ```
72//! The ability to use the key:value system with the log crate's macros should come up with the 0.4.15 release or afterwards.
73//!
74//! The kv_unstable feature allows you to use the [`log`](https://crates.io/crates/log) facade as such:
75//!
76//! ```ignore
77//! # extern crate log;
78//! # extern crate loki_logger;
79//! # use std::iter::FromIterator;
80//! use std::collections::HashMap;
81//! use log::LevelFilter;
82//!
83//! # #[tokio::main]
84//! # async fn main() {
85//!
86//! loki_logger::init(
87//!     "http://loki:3100/loki/api/v1/push",
88//!     log::LevelFilter::Info,
89//! ).unwrap();
90//!
91//! // Due to stabilization issue, this is still unstable,
92//! // the log macros needs to have at least one formatting parameter for this to work.
93//! log::info!(foo = "bar"; "Logged into Loki !{}", "");
94//! # }
95//! ```
96//!
97//! # Notice on asynchronous execution
98//!
99//! The loki_logger crate ships with asynchronous execution, orchestrated with [`tokio`](https://tokio.rs/), by default.
100//!
101//! This means that for the logging operations to work, you need to be in the scope of a asynchronous runtime first.
102//!
103//! Otherwise, you can activate the `blocking` feature of The loki_logger crate to use a blocking client.
104//!
105//! THIS IS NOT RECOMMENDED FOR PRODUCTIONS WORKLOAD.
106
107use serde::Serialize;
108use std::{
109    collections::HashMap,
110    error::Error,
111    time::{SystemTime, UNIX_EPOCH},
112};
113
114use log::{
115    kv::{Source, Visitor},
116    LevelFilter, Metadata, Record, SetLoggerError,
117};
118
119/// Re-export of the log crate for use with a different version by the `loki-logger` crate's user.
120pub use log;
121
122#[derive(Serialize)]
123struct LokiStream {
124    stream: HashMap<String, String>,
125    values: Vec<[String; 2]>,
126}
127
128#[derive(Serialize)]
129struct LokiRequest {
130    streams: Vec<LokiStream>,
131}
132
133#[cfg(not(feature = "blocking"))]
134struct LokiLogger {
135    url: String,
136    initial_labels: Option<HashMap<String, String>>,
137    client: reqwest::Client,
138}
139
140#[cfg(feature = "blocking")]
141struct LokiLogger {
142    url: String,
143    initial_labels: Option<HashMap<String, String>>,
144    client: reqwest::blocking::Client,
145}
146
147fn init_inner<S: AsRef<str>>(
148    url: S,
149    max_log_level: LevelFilter,
150    initial_labels: Option<HashMap<String, String>>,
151) -> Result<(), SetLoggerError> {
152    let logger = Box::new(LokiLogger::new(url, initial_labels));
153    log::set_boxed_logger(logger).map(|()| log::set_max_level(max_log_level))
154}
155
156/// Configure the [`log`](https://crates.io/crates/log) facade to log to [loki](https://grafana.com/oss/loki/).
157///
158/// This function initialize the logger with no defaults [static labels](https://grafana.com/docs/loki/latest/best-practices/#static-labels-are-good).
159/// To use them, you may want to use [`init_with_labels`].
160///
161/// # Example
162///
163/// Usage:
164///
165/// ```rust
166/// # extern crate log;
167/// # extern crate loki_logger;
168/// use log::LevelFilter;
169///
170/// # #[tokio::main]
171/// # async fn main() {
172/// loki_logger::init(
173///     "http://loki:3100/loki/api/v1/push",
174///     log::LevelFilter::Info,
175/// ).unwrap();
176///
177/// log::info!("Logged into Loki !");
178/// # }
179/// ```
180pub fn init<S: AsRef<str>>(url: S, max_log_level: LevelFilter) -> Result<(), SetLoggerError> {
181    init_inner(url, max_log_level, None)
182}
183
184/// Configure the [`log`](https://crates.io/crates/log) facade to log to [loki](https://grafana.com/oss/loki/).
185///
186/// This function initialize the logger with defaults [static labels](https://grafana.com/docs/loki/latest/best-practices/#static-labels-are-good).
187/// To not use them, you may want to use [`init`].
188///
189/// # Example
190///
191/// Usage:
192///
193/// ```rust
194/// # extern crate log;
195/// # extern crate loki_logger;
196/// # use std::iter::FromIterator;
197/// use std::collections::HashMap;
198/// use log::LevelFilter;
199///
200/// # #[tokio::main]
201/// # async fn main() {
202/// let initial_labels = HashMap::from_iter([
203///     ("application".to_string(), "loki_logger".to_string()),
204///     ("environment".to_string(), "development".to_string()),
205/// ]);
206///
207/// loki_logger::init_with_labels(
208///     "http://loki:3100/loki/api/v1/push",
209///     log::LevelFilter::Info,
210///     initial_labels
211/// ).unwrap();
212///
213/// log::info!("Logged into Loki !");
214/// # }
215/// ```
216pub fn init_with_labels<S: AsRef<str>>(
217    url: S,
218    max_log_level: LevelFilter,
219    initial_labels: HashMap<String, String>,
220) -> Result<(), SetLoggerError> {
221    init_inner(url, max_log_level, Some(initial_labels))
222}
223
224struct LokiVisitor<'kvs> {
225    values: HashMap<log::kv::Key<'kvs>, log::kv::Value<'kvs>>,
226}
227
228impl<'kvs> LokiVisitor<'kvs> {
229    pub fn new(count: usize) -> Self {
230        Self {
231            values: HashMap::with_capacity(count),
232        }
233    }
234
235    pub fn read_kv(
236        &'kvs mut self,
237        source: &'kvs dyn Source,
238    ) -> Result<&HashMap<log::kv::Key<'kvs>, log::kv::Value<'kvs>>, log::kv::Error> {
239        for _ in 0..source.count() {
240            source.visit(self)?;
241        }
242        Ok(&self.values)
243    }
244}
245
246impl<'kvs> Visitor<'kvs> for LokiVisitor<'kvs> {
247    fn visit_pair(
248        &mut self,
249        key: log::kv::Key<'kvs>,
250        value: log::kv::Value<'kvs>,
251    ) -> Result<(), log::kv::Error> {
252        self.values.insert(key, value);
253        Ok(())
254    }
255}
256
257impl log::Log for LokiLogger {
258    fn enabled(&self, _: &Metadata) -> bool {
259        true
260    }
261
262    fn log(&self, record: &Record) {
263        if self.enabled(record.metadata()) {
264            if let Err(e) = self.log_event_record(record) {
265                eprintln!("Impossible to log event to loki: {:?}", e)
266            }
267        }
268    }
269
270    fn flush(&self) {}
271}
272
273impl LokiLogger {
274    #[cfg(not(feature = "blocking"))]
275    fn new<S: AsRef<str>>(url: S, initial_labels: Option<HashMap<String, String>>) -> Self {
276        Self {
277            url: url.as_ref().to_string(),
278            initial_labels,
279            client: reqwest::Client::new(),
280        }
281    }
282
283    #[cfg(feature = "blocking")]
284    fn new<S: AsRef<str>>(url: S, initial_labels: Option<HashMap<String, String>>) -> Self {
285        Self {
286            url: url.as_ref().to_string(),
287            initial_labels,
288            client: reqwest::blocking::Client::new(),
289        }
290    }
291
292    #[cfg(not(feature = "blocking"))]
293    fn log_to_loki(
294        &self,
295        message: String,
296        labels: HashMap<String, String>,
297    ) -> Result<(), Box<dyn Error>> {
298        let client = self.client.clone();
299        let url = self.url.clone();
300
301        let loki_request = make_request(message, labels)?;
302        tokio::spawn(async move {
303            if let Err(e) = client.post(url).json(&loki_request).send().await {
304                eprintln!("{:?}", e);
305            };
306        });
307        Ok(())
308    }
309
310    #[cfg(feature = "blocking")]
311    fn log_to_loki(
312        &self,
313        message: String,
314        labels: HashMap<String, String>,
315    ) -> Result<(), Box<dyn Error>> {
316        let url = self.url.clone();
317
318        let loki_request = make_request(message, labels)?;
319        self.client.post(url).json(&loki_request).send()?;
320        Ok(())
321    }
322
323    fn merge_loki_labels(
324        &self,
325        kv_labels: &HashMap<log::kv::Key, log::kv::Value>,
326    ) -> HashMap<String, String> {
327        merge_labels(self.initial_labels.as_ref(), kv_labels)
328    }
329
330    fn log_event_record(&self, record: &Record) -> Result<(), Box<dyn Error>> {
331        let kv = record.key_values();
332        let mut visitor = LokiVisitor::new(kv.count());
333        let values = visitor.read_kv(kv)?;
334        let message = format!("{:?}", record.args());
335        let mut labels = self.merge_loki_labels(values);
336        labels.insert(
337            "level".to_string(),
338            record.level().to_string().to_ascii_lowercase(),
339        );
340        self.log_to_loki(message, labels)
341    }
342}
343
344fn merge_labels(
345    initial_labels: Option<&HashMap<String, String>>,
346    kv_labels: &HashMap<log::kv::Key, log::kv::Value>,
347) -> HashMap<String, String> {
348    let mut labels = if let Some(initial_labels) = initial_labels {
349        initial_labels.clone()
350    } else {
351        HashMap::with_capacity(kv_labels.len())
352    };
353    labels.extend(
354        kv_labels
355            .iter()
356            .map(|(key, value)| (key.to_string(), value.to_string())),
357    );
358    labels
359}
360
361fn make_request(
362    message: String,
363    labels: HashMap<String, String>,
364) -> Result<LokiRequest, Box<dyn Error>> {
365    let start = SystemTime::now();
366    let time_ns = time_offset_since(start)?;
367    let loki_request = LokiRequest {
368        streams: vec![LokiStream {
369            stream: labels,
370            values: vec![[time_ns, message]],
371        }],
372    };
373    Ok(loki_request)
374}
375
376fn time_offset_since(start: SystemTime) -> Result<String, Box<dyn Error>> {
377    let since_start = start.duration_since(UNIX_EPOCH)?;
378    let time_ns = since_start.as_nanos().to_string();
379    Ok(time_ns)
380}
381
382#[cfg(test)]
383mod tests {
384    use log::kv::{Key, Value};
385
386    use crate::{merge_labels, time_offset_since};
387    use std::{
388        collections::HashMap,
389        time::{Duration, SystemTime},
390    };
391
392    #[test]
393    fn time_offsets() {
394        let t1 = time_offset_since(SystemTime::now());
395        assert!(t1.is_ok());
396
397        // Constructing a negative timestamp
398        let negative_time = SystemTime::UNIX_EPOCH.checked_sub(Duration::from_secs(1));
399
400        assert!(negative_time.is_some());
401
402        let t2 = time_offset_since(negative_time.unwrap());
403        assert!(t2.is_err());
404    }
405
406    #[test]
407    fn merge_no_initial_labels() {
408        let kv_labels = HashMap::new();
409        let merged_labels = merge_labels(None, &kv_labels);
410
411        assert_eq!(merged_labels, HashMap::new());
412
413        let kv_labels = [
414            (Key::from_str("application"), Value::from("loki_logger")),
415            (Key::from_str("environment"), Value::from("development")),
416        ]
417        .into_iter()
418        .collect::<HashMap<_, _>>();
419        let merged_labels = merge_labels(None, &kv_labels);
420
421        assert_eq!(
422            merged_labels,
423            [
424                ("application".to_string(), "loki_logger".to_string()),
425                ("environment".to_string(), "development".to_string())
426            ]
427            .into_iter()
428            .collect::<HashMap<_, _>>()
429        );
430    }
431
432    #[test]
433    fn merge_initial_labels() {
434        let kv_labels = HashMap::new();
435        let initial_labels = HashMap::new();
436        let merged_labels = merge_labels(Some(&initial_labels), &kv_labels);
437
438        assert_eq!(merged_labels, HashMap::new());
439
440        let kv_labels = HashMap::new();
441        let initial_labels = [
442            ("application".to_string(), "loki_logger".to_string()),
443            ("environment".to_string(), "development".to_string()),
444        ]
445        .into_iter()
446        .collect::<HashMap<_, _>>();
447        let merged_labels = merge_labels(Some(&initial_labels), &kv_labels);
448
449        assert_eq!(merged_labels, initial_labels);
450
451        let initial_labels = [
452            ("application".to_string(), "loki_logger".to_string()),
453            ("environment".to_string(), "development".to_string()),
454        ]
455        .into_iter()
456        .collect::<HashMap<_, _>>();
457        let kv_labels = [
458            (Key::from_str("event_name"), Value::from("request")),
459            (Key::from_str("handler"), Value::from("/loki/api/v1/push")),
460        ]
461        .into_iter()
462        .collect::<HashMap<_, _>>();
463        let merged_labels = merge_labels(Some(&initial_labels), &kv_labels);
464
465        assert_eq!(
466            merged_labels,
467            [
468                ("application".to_string(), "loki_logger".to_string()),
469                ("environment".to_string(), "development".to_string()),
470                ("event_name".to_string(), "request".to_string()),
471                ("handler".to_string(), "/loki/api/v1/push".to_string())
472            ]
473            .into_iter()
474            .collect::<HashMap<_, _>>()
475        );
476    }
477
478    #[test]
479    fn merge_overwrite_labels() {
480        let initial_labels = [
481            ("application".to_string(), "loki_logger".to_string()),
482            ("environment".to_string(), "development".to_string()),
483        ]
484        .into_iter()
485        .collect::<HashMap<_, _>>();
486        let kv_labels = [
487            (Key::from_str("event_name"), Value::from("request")),
488            (Key::from_str("environment"), Value::from("production")),
489        ]
490        .into_iter()
491        .collect::<HashMap<_, _>>();
492        let merged_labels = merge_labels(Some(&initial_labels), &kv_labels);
493
494        assert_eq!(
495            merged_labels,
496            [
497                ("application".to_string(), "loki_logger".to_string()),
498                ("environment".to_string(), "production".to_string()),
499                ("event_name".to_string(), "request".to_string()),
500            ]
501            .into_iter()
502            .collect::<HashMap<_, _>>()
503        );
504    }
505}
loki_logger/lib.rs

loki_logger/
lib.rs
