gst_log_parser/
lib.rs

1// Copyright (C) 2017-2019 Guillaume Desmottes <guillaume@desmottes.be>
2//
3// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
4// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
5// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
6// option. This file may not be copied, modified, or distributed
7// except according to those terms.
8
9use itertools::join;
10use std::fmt;
11use std::io::BufRead;
12use std::io::BufReader;
13use std::io::Lines;
14use std::io::Read;
15use std::str;
16use std::str::FromStr;
17
18use gst::{ClockTime, DebugLevel, Structure};
19use gstreamer as gst;
20use lazy_static::lazy_static;
21use regex::Regex;
22use thiserror::Error;
23
24#[derive(Debug, PartialEq)]
25pub enum TimestampField {
26    Hour,
27    Minute,
28    Second,
29    SubSecond,
30}
31
32#[derive(Debug, PartialEq)]
33pub enum Token {
34    Timestamp { field: Option<TimestampField> },
35    PID,
36    Thread,
37    Level,
38    Category,
39    File,
40    LineNumber,
41    Function,
42    Message,
43    Object,
44}
45
46#[derive(Debug, Error, PartialEq)]
47pub enum ParsingError {
48    #[error("invalid debug level: {}", name)]
49    InvalidDebugLevel { name: String },
50    #[error("invalid timestamp: {} : {:?}", ts, field)]
51    InvalidTimestamp { ts: String, field: TimestampField },
52    #[error("missing token: {:?}", t)]
53    MissingToken { t: Token },
54    #[error("invalid PID: {}", pid)]
55    InvalidPID { pid: String },
56    #[error("missing location")]
57    MissingLocation,
58    #[error("invalid line number: {}", line)]
59    InvalidLineNumber { line: String },
60}
61
62#[derive(Debug)]
63pub struct Entry {
64    pub ts: ClockTime,
65    pub pid: u32,
66    pub thread: String,
67    pub level: DebugLevel,
68    pub category: String,
69    pub file: String,
70    pub line: u32,
71    pub function: String,
72    pub message: String,
73    pub object: Option<String>,
74}
75
76fn parse_debug_level(s: &str) -> Result<DebugLevel, ParsingError> {
77    match s {
78        "ERROR" => Ok(DebugLevel::Error),
79        "WARN" => Ok(DebugLevel::Warning),
80        "FIXME" => Ok(DebugLevel::Fixme),
81        "INFO" => Ok(DebugLevel::Info),
82        "DEBUG" => Ok(DebugLevel::Debug),
83        "LOG" => Ok(DebugLevel::Log),
84        "TRACE" => Ok(DebugLevel::Trace),
85        "MEMDUMP" => Ok(DebugLevel::Memdump),
86        _ => Err(ParsingError::InvalidDebugLevel {
87            name: s.to_string(),
88        }),
89    }
90}
91
92fn parse_time(ts: &str) -> Result<ClockTime, ParsingError> {
93    let mut split = ts.splitn(3, ':');
94    let h: u64 = split
95        .next()
96        .ok_or(ParsingError::MissingToken {
97            t: Token::Timestamp {
98                field: Some(TimestampField::Hour),
99            },
100        })?
101        .parse()
102        .map_err(|_e| ParsingError::InvalidTimestamp {
103            ts: ts.to_string(),
104            field: TimestampField::Hour,
105        })?;
106
107    let m: u64 = split
108        .next()
109        .ok_or(ParsingError::MissingToken {
110            t: Token::Timestamp {
111                field: Some(TimestampField::Minute),
112            },
113        })?
114        .parse()
115        .map_err(|_e| ParsingError::InvalidTimestamp {
116            ts: ts.to_string(),
117            field: TimestampField::Minute,
118        })?;
119
120    split = split
121        .next()
122        .ok_or(ParsingError::MissingToken {
123            t: Token::Timestamp {
124                field: Some(TimestampField::Second),
125            },
126        })?
127        .splitn(2, '.');
128    let secs: u64 = split
129        .next()
130        .ok_or(ParsingError::MissingToken {
131            t: Token::Timestamp {
132                field: Some(TimestampField::Second),
133            },
134        })?
135        .parse()
136        .map_err(|_e| ParsingError::InvalidTimestamp {
137            ts: ts.to_string(),
138            field: TimestampField::Second,
139        })?;
140
141    let subsecs: u64 = split
142        .next()
143        .ok_or(ParsingError::MissingToken {
144            t: Token::Timestamp {
145                field: Some(TimestampField::SubSecond),
146            },
147        })?
148        .parse()
149        .map_err(|_e| ParsingError::InvalidTimestamp {
150            ts: ts.to_string(),
151            field: TimestampField::SubSecond,
152        })?;
153
154    Ok(ClockTime::from_seconds(h * 60 * 60 + m * 60 + secs) + ClockTime::from_nseconds(subsecs))
155}
156
157fn split_location(location: &str) -> Result<(String, u32, String, Option<String>), ParsingError> {
158    let mut split = location.splitn(4, ':');
159    let file = split
160        .next()
161        .ok_or(ParsingError::MissingToken { t: Token::File })?;
162    let line_str = split.next().ok_or(ParsingError::MissingToken {
163        t: Token::LineNumber,
164    })?;
165    let line = line_str
166        .parse()
167        .map_err(|_e| ParsingError::InvalidLineNumber {
168            line: line_str.to_string(),
169        })?;
170
171    let function = split
172        .next()
173        .ok_or(ParsingError::MissingToken { t: Token::Function })?;
174
175    let object = split
176        .next()
177        .ok_or(ParsingError::MissingToken { t: Token::Object })?;
178
179    let object_name = {
180        if !object.is_empty() {
181            let object = object
182                .to_string()
183                .trim_start_matches('<')
184                .trim_end_matches('>')
185                .to_string();
186
187            Some(object)
188        } else {
189            None
190        }
191    };
192
193    Ok((file.to_string(), line, function.to_string(), object_name))
194}
195
196impl Entry {
197    fn new(line: &str) -> Result<Entry, ParsingError> {
198        // Strip color codes
199        lazy_static! {
200            static ref RE: Regex = Regex::new("\x1b\\[[0-9;]*m").unwrap();
201        }
202        let line = RE.replace_all(line, "");
203
204        let mut it = line.split(' ');
205        let ts_str = it.next().ok_or(ParsingError::MissingToken {
206            t: Token::Timestamp { field: None },
207        })?;
208        let ts = parse_time(ts_str)?;
209
210        let mut it = it.skip_while(|x| x.is_empty());
211        let pid_str = it
212            .next()
213            .ok_or(ParsingError::MissingToken { t: Token::PID })?;
214        let pid = pid_str.parse().map_err(|_e| ParsingError::InvalidPID {
215            pid: pid_str.to_string(),
216        })?;
217
218        let mut it = it.skip_while(|x| x.is_empty());
219        let thread = it
220            .next()
221            .ok_or(ParsingError::MissingToken { t: Token::Thread })?
222            .to_string();
223
224        let mut it = it.skip_while(|x| x.is_empty());
225        let level_str = it
226            .next()
227            .ok_or(ParsingError::MissingToken { t: Token::Level })?;
228        let level = parse_debug_level(level_str)?;
229
230        let mut it = it.skip_while(|x| x.is_empty());
231        let category = it
232            .next()
233            .ok_or(ParsingError::MissingToken { t: Token::Category })?
234            .to_string();
235
236        let mut it = it.skip_while(|x| x.is_empty());
237        let location_str = it.next().ok_or(ParsingError::MissingLocation)?;
238        let (file, line, function, object) = split_location(location_str)?;
239        let message: String = join(it, " ");
240
241        Ok(Entry {
242            ts,
243            pid,
244            thread,
245            level,
246            category,
247            file,
248            line,
249            function,
250            object,
251            message,
252        })
253    }
254
255    pub fn message_to_struct(&self) -> Option<Structure> {
256        Structure::from_str(&self.message).ok()
257    }
258}
259
260impl fmt::Display for Entry {
261    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
262        write!(
263            f,
264            "{}  {} {} {:?} {} {}:{}:{}:<{}> {}",
265            self.ts,
266            self.pid,
267            self.thread,
268            self.level,
269            self.category,
270            self.file,
271            self.line,
272            self.function,
273            self.object.clone().unwrap_or_default(),
274            self.message
275        )
276    }
277}
278
279pub struct ParserIterator<R: Read> {
280    lines: Lines<BufReader<R>>,
281}
282
283impl<R: Read> ParserIterator<R> {
284    fn new(lines: Lines<BufReader<R>>) -> Self {
285        Self { lines }
286    }
287}
288
289impl<R: Read> Iterator for ParserIterator<R> {
290    type Item = Entry;
291
292    fn next(&mut self) -> Option<Entry> {
293        match self.lines.next() {
294            None => None,
295            Some(line) => match Entry::new(&line.unwrap()) {
296                Ok(entry) => Some(entry),
297                Err(_err) => self.next(),
298            },
299        }
300    }
301}
302
303pub fn parse<R: Read>(r: R) -> ParserIterator<R> {
304    gst::init().expect("Failed to initialize gst");
305
306    let file = BufReader::new(r);
307
308    ParserIterator::new(file.lines())
309}
310
311#[cfg(test)]
312mod tests {
313    use super::*;
314    use std::fs::File;
315
316    #[test]
317    fn no_color() {
318        let f = File::open("test-logs/nocolor.log").expect("Failed to open log file");
319        let mut parsed = parse(f);
320
321        let entry = parsed.next().expect("First entry missing");
322        assert_eq!(entry.ts.nseconds(), 7773544);
323        assert_eq!(format!("{}", entry.ts), "0:00:00.007773544");
324        assert_eq!(entry.pid, 8874);
325        assert_eq!(entry.thread, "0x558951015c00");
326        assert_eq!(entry.level, DebugLevel::Info);
327        assert_eq!(entry.category, "GST_INIT");
328        assert_eq!(entry.file, "gst.c");
329        assert_eq!(entry.line, 510);
330        assert_eq!(entry.function, "init_pre");
331        assert_eq!(
332            entry.message,
333            "Initializing GStreamer Core Library version 1.10.4"
334        );
335
336        let entry = parsed.nth(3).expect("3th entry missing");
337        assert_eq!(entry.message, "0x55895101d040 ref 1->2");
338        assert_eq!(entry.object, Some("allocatorsysmem0".to_string()));
339    }
340
341    fn parse_file(f: File) -> (Entry, usize) {
342        let mut parsed = parse(f);
343
344        let entry = parsed.next().expect("First entry missing");
345        (entry, parsed.count() + 1)
346    }
347
348    #[test]
349    fn color() {
350        let f = File::open("test-logs/color.log").expect("Failed to open log file");
351        let (entry, count) = parse_file(f);
352        assert_eq!(entry.ts.nseconds(), 208614);
353        assert_eq!(format!("{}", entry.ts), "0:00:00.000208614");
354        assert_eq!(entry.pid, 17267);
355        assert_eq!(entry.thread, "0x2192200");
356        assert_eq!(entry.level, DebugLevel::Info);
357        assert_eq!(entry.category, "GST_INIT");
358        assert_eq!(entry.file, "gst.c");
359        assert_eq!(entry.line, 584);
360        assert_eq!(entry.function, "init_pre");
361        assert_eq!(
362            entry.message,
363            "Initializing GStreamer Core Library version 1.13.0.1"
364        );
365        assert_eq!(count, 15);
366    }
367
368    #[test]
369    fn corrupted() {
370        let f = File::open("test-logs/corrupted-nocolor.log").expect("Failed to open log file");
371        let (entry, count) = parse_file(f);
372
373        assert_eq!(entry.ts.nseconds(), 7773544);
374        assert_eq!(format!("{}", entry.ts), "0:00:00.007773544");
375        assert_eq!(entry.pid, 8874);
376        assert_eq!(entry.thread, "0x558951015c00");
377        assert_eq!(entry.level, DebugLevel::Info);
378        assert_eq!(entry.category, "GST_INIT");
379        assert_eq!(entry.file, "gst.c");
380        assert_eq!(entry.line, 510);
381        assert_eq!(entry.function, "init_pre");
382        assert_eq!(
383            entry.message,
384            "Initializing GStreamer Core Library version 1.10.4"
385        );
386        assert_eq!(count, 6);
387    }
388
389    #[test]
390    fn parse_corrupted() {
391        let f = File::open("test-logs/corrupted-nocolor.log").expect("Failed to open log file");
392        let (_entry, count) = parse_file(f);
393        assert!(count > 0);
394    }
395
396    #[test]
397    fn timestamps() {
398        assert!(Entry::new("foo").is_err());
399
400        let e1 = "e:00:00.007773544  8874 0x558951015c00 INFO                GST_INIT gst.c:510:init_pre: Init";
401        match Entry::new(e1) {
402            Ok(_) => unreachable!(),
403            Err(e) => assert_eq!(
404                e,
405                ParsingError::InvalidTimestamp {
406                    ts: "e:00:00.007773544".to_string(),
407                    field: TimestampField::Hour,
408                }
409            ),
410        };
411
412        let e1 = ":00:00.007773544  8874 0x558951015c00 INFO                GST_INIT gst.c:510:init_pre: Init";
413        match Entry::new(e1) {
414            Ok(_) => unreachable!(),
415            Err(e) => assert_eq!(
416                e,
417                ParsingError::InvalidTimestamp {
418                    ts: ":00:00.007773544".to_string(),
419                    field: TimestampField::Hour,
420                }
421            ),
422        };
423
424        let e1 = "8874 0x558951015c00 INFO                GST_INIT gst.c:510:init_pre: Init";
425        match Entry::new(e1) {
426            Ok(_) => unreachable!(),
427            Err(e) => assert_eq!(
428                e,
429                ParsingError::MissingToken {
430                    t: Token::Timestamp {
431                        field: Some(TimestampField::Minute)
432                    },
433                }
434            ),
435        };
436    }
437
438    #[test]
439    fn pid() {
440        let e1 = "00:00:00.007773544 ";
441        match Entry::new(e1) {
442            Ok(_) => unreachable!(),
443            Err(e) => assert_eq!(e, ParsingError::MissingToken { t: Token::PID }),
444        };
445
446        let e1 = "00:00:00.007773544  8fuz874 0x558951015c00 INFO                GST_INIT gst.c:510:init_pre: Init";
447        match Entry::new(e1) {
448            Ok(_) => unreachable!(),
449            Err(e) => assert_eq!(
450                e,
451                ParsingError::InvalidPID {
452                    pid: "8fuz874".to_string(),
453                }
454            ),
455        };
456    }
457
458    #[test]
459    fn thread() {
460        let e1 = "00:00:00.007773544  8874 ";
461        match Entry::new(e1) {
462            Ok(_) => unreachable!(),
463            Err(e) => assert_eq!(e, ParsingError::MissingToken { t: Token::Thread }),
464        };
465    }
466
467    #[test]
468    fn debug_level() {
469        let e1 = "00:00:00.007773544  8874 0x558951015c00 ";
470        match Entry::new(e1) {
471            Ok(_) => unreachable!(),
472            Err(e) => assert_eq!(e, ParsingError::MissingToken { t: Token::Level }),
473        };
474
475        let e1 = "00:00:00.007773544  8874 0x558951015c00 FUZZLEVEL                GST_INIT gst.c:510:init_pre: Init";
476        match Entry::new(e1) {
477            Ok(_) => unreachable!(),
478            Err(e) => assert_eq!(
479                e,
480                ParsingError::InvalidDebugLevel {
481                    name: "FUZZLEVEL".to_string(),
482                }
483            ),
484        };
485    }
486
487    #[test]
488    fn category() {
489        let e1 = "00:00:00.007773544  8874 0x558951015c00 INFO";
490        match Entry::new(e1) {
491            Ok(_) => unreachable!(),
492            Err(e) => assert_eq!(e, ParsingError::MissingToken { t: Token::Category }),
493        };
494    }
495
496    #[test]
497    fn location() {
498        let e1 = "00:00:00.007773544  8874 0x558951015c00 INFO GST_INIT";
499        match Entry::new(e1) {
500            Ok(_) => unreachable!(),
501            Err(e) => assert_eq!(e, ParsingError::MissingLocation {}),
502        };
503
504        let e1 = "00:00:00.007773544  8874 0x558951015c00 INFO GST_INIT gst.c";
505        match Entry::new(e1) {
506            Ok(_) => unreachable!(),
507            Err(e) => assert_eq!(
508                e,
509                ParsingError::MissingToken {
510                    t: Token::LineNumber
511                }
512            ),
513        };
514
515        let e1 = "00:00:00.007773544  8874 0x558951015c00 INFO GST_INIT gst.c:fuzz";
516        match Entry::new(e1) {
517            Ok(_) => unreachable!(),
518            Err(e) => assert_eq!(
519                e,
520                ParsingError::InvalidLineNumber {
521                    line: "fuzz".to_string()
522                }
523            ),
524        };
525
526        let e1 = "00:00:00.007773544  8874 0x558951015c00 INFO GST_INIT gst.c:510";
527        match Entry::new(e1) {
528            Ok(_) => unreachable!(),
529            Err(e) => assert_eq!(e, ParsingError::MissingToken { t: Token::Function }),
530        };
531
532        let e1 = "00:00:00.007773544  8874 0x558951015c00 INFO GST_INIT gst.c:510:";
533        match Entry::new(e1) {
534            Ok(_) => unreachable!(),
535            Err(e) => assert_eq!(e, ParsingError::MissingToken { t: Token::Object }),
536        };
537    }
538}
gst_log_parser/lib.rs

gst_log_parser/
lib.rs
