Skip to main content

cmtraceopen_parser/parser/
cmtlog.rs

1//! CmtLog format parser.
2//!
3//! Extends CCM's `<![LOG[...]LOG]!>` line format with reserved component names
4//! (`__HEADER__`, `__SECTION__`, `__ITERATION__`) and optional extended attributes
5//! (`section`, `tag`, `whatif`, `iteration`, `color`).
6//!
7//! Uses a relaxed CCM regex that allows trailing key="value" attributes after the
8//! standard `file=""` field, then post-processes entries to extract extended
9//! attributes and classify by component name.
10
11use regex::Regex;
12use std::sync::OnceLock;
13
14use super::ccm;
15use super::severity::detect_severity_from_text;
16use crate::models::log_entry::{EntryKind, LogEntry, LogFormat, Severity};
17
18/// Reserved component names that signal CmtLog structured entries.
19const HEADER_COMPONENT: &str = "__HEADER__";
20const SECTION_COMPONENT: &str = "__SECTION__";
21const ITERATION_COMPONENT: &str = "__ITERATION__";
22
23/// Compiled regex for extracting key="value" pairs from raw log lines.
24fn attr_re() -> &'static Regex {
25    static CELL: OnceLock<Regex> = OnceLock::new();
26    CELL.get_or_init(|| Regex::new(r#"(\w+)="([^"]*)""#).expect("attr regex must compile"))
27}
28
29/// Relaxed CCM regex that allows arbitrary trailing attributes after `file=""`.
30///
31/// Standard CCM regex requires `>` immediately after the optional `file` field.
32/// CmtLog lines contain additional key="value" pairs (script, version, color,
33/// section, etc.) between `file=""` and `>`.  This regex uses `[^>]*>` to
34/// tolerate those extra attributes.
35fn cmtlog_re() -> &'static Regex {
36    static CELL: OnceLock<Regex> = OnceLock::new();
37    CELL.get_or_init(|| {
38        Regex::new(concat!(
39            r#"<!\[LOG\[(?P<msg>[\s\S]*?)\]LOG\]!>"#,
40            r#"<time="(?P<h>\d{1,2}):(?P<m>\d{1,2}):(?P<s>\d{1,2})\.(?P<ms>\d+)(?P<tz>[+-]*\d+)""#,
41            r#"\s+date="(?P<mon>\d{1,2})-(?P<day>\d{1,2})-(?P<yr>\d{4})""#,
42            r#"\s+component="(?P<comp>[^"]*)""#,
43            r#"\s+context="[^"]*""#,
44            r#"\s+type="(?P<typ>\d)""#,
45            r#"\s+thread="(?P<thr>\d+)""#,
46            r#"[^>]*>"#,
47        ))
48        .expect("CmtLog regex must compile")
49    })
50}
51
52/// Returns true if the line contains a CmtLog reserved `component="..."` attribute,
53/// indicating the file uses the CmtLog format rather than plain CCM.
54///
55/// Checks for exact attribute syntax to avoid false positives when reserved
56/// names appear in log message text.
57pub fn matches_cmtlog_record(line: &str) -> bool {
58    line.contains(&format!("component=\"{}\"", HEADER_COMPONENT))
59        || line.contains(&format!("component=\"{}\"", SECTION_COMPONENT))
60        || line.contains(&format!("component=\"{}\"", ITERATION_COMPONENT))
61}
62
63/// Extract extended attributes from a raw CmtLog line.
64///
65/// Scans for `key="value"` pairs and returns the ones relevant to CmtLog:
66/// `section`, `color`, `tag`, `whatif`, `iteration`.
67fn extract_attrs(line: &str) -> ExtractedAttrs {
68    let mut attrs = ExtractedAttrs::default();
69    for caps in attr_re().captures_iter(line) {
70        let key = caps.get(1).map(|m| m.as_str()).unwrap_or("");
71        let value = caps.get(2).map(|m| m.as_str()).unwrap_or("");
72        match key {
73            "section" => attrs.section = Some(value.to_string()),
74            "color" => attrs.color = Some(value.to_string()),
75            "tag" => attrs.tag = Some(value.to_string()),
76            "whatif" => attrs.whatif = Some(value.to_string()),
77            "iteration" => attrs.iteration = Some(value.to_string()),
78            _ => {}
79        }
80    }
81    attrs
82}
83
84#[derive(Default)]
85struct ExtractedAttrs {
86    section: Option<String>,
87    color: Option<String>,
88    tag: Option<String>,
89    whatif: Option<String>,
90    iteration: Option<String>,
91}
92
93/// Parse a single CmtLog line using the relaxed regex.
94fn parse_cmtlog_line(line: &str) -> Option<CmtLogParsed> {
95    let caps = cmtlog_re().captures(line)?;
96
97    let message = caps.name("msg").map(|m| m.as_str().to_string())?;
98    let component_str = caps.name("comp").map(|m| m.as_str().to_string());
99    let severity_type = caps
100        .name("typ")
101        .and_then(|m| m.as_str().parse::<u32>().ok());
102    let severity = ccm::severity_from_type_field(severity_type, &message);
103
104    let thread = caps
105        .name("thr")
106        .and_then(|m| m.as_str().parse::<u32>().ok())
107        .unwrap_or(0);
108    let thread_display = Some(ccm::format_thread_display(thread));
109
110    let h: u32 = caps
111        .name("h")
112        .and_then(|m| m.as_str().parse().ok())
113        .unwrap_or(0);
114    let min: u32 = caps
115        .name("m")
116        .and_then(|m| m.as_str().parse().ok())
117        .unwrap_or(0);
118    let s: u32 = caps
119        .name("s")
120        .and_then(|m| m.as_str().parse().ok())
121        .unwrap_or(0);
122    let ms: u32 = caps
123        .name("ms")
124        .and_then(|m| m.as_str().parse().ok())
125        .unwrap_or(0);
126    let tz_str = caps.name("tz").map(|m| m.as_str()).unwrap_or("0");
127    let tz_offset: i32 = tz_str.replace("+-", "-").parse().unwrap_or(0);
128    let mon: u32 = caps
129        .name("mon")
130        .and_then(|m| m.as_str().parse().ok())
131        .unwrap_or(1);
132    let day: u32 = caps
133        .name("day")
134        .and_then(|m| m.as_str().parse().ok())
135        .unwrap_or(1);
136    let yr: i32 = caps
137        .name("yr")
138        .and_then(|m| m.as_str().parse().ok())
139        .unwrap_or(2000);
140
141    let (timestamp, timestamp_display) =
142        ccm::build_timestamp(mon, day, yr, h, min, s, ms, Some(tz_offset));
143
144    Some(CmtLogParsed {
145        message,
146        component: component_str,
147        timestamp,
148        timestamp_display,
149        severity,
150        thread,
151        thread_display,
152        timezone_offset: tz_offset,
153    })
154}
155
156struct CmtLogParsed {
157    message: String,
158    component: Option<String>,
159    timestamp: Option<i64>,
160    timestamp_display: Option<String>,
161    severity: Severity,
162    thread: u32,
163    thread_display: Option<String>,
164    timezone_offset: i32,
165}
166
167/// Parse all lines as CmtLog format.
168///
169/// Uses a relaxed CCM regex that tolerates extra key="value" attributes, then
170/// post-processes entries to extract CmtLog-specific attributes and classify
171/// by component name.
172///
173/// Returns `(entries, parse_error_count)`.
174pub fn parse_lines(lines: &[&str], file_path: &str) -> (Vec<LogEntry>, u32) {
175    let mut entries = Vec::with_capacity(lines.len());
176    let mut errors = 0u32;
177    let mut id_counter = 0u64;
178
179    // Track current section context for propagation to child entries.
180    let mut current_section_name: Option<String> = None;
181    let mut current_section_color: Option<String> = None;
182
183    for (i, line) in lines.iter().enumerate() {
184        if line.trim().is_empty() {
185            continue;
186        }
187
188        let attrs = extract_attrs(line);
189
190        let (base_entry, component_str) = match parse_cmtlog_line(line) {
191            Some(parsed) => {
192                let comp = parsed.component.clone();
193                let entry = LogEntry {
194                    id: id_counter,
195                    line_number: (i + 1) as u32,
196                    message: parsed.message,
197                    component: parsed.component,
198                    timestamp: parsed.timestamp,
199                    timestamp_display: parsed.timestamp_display,
200                    severity: parsed.severity,
201                    thread: Some(parsed.thread),
202                    thread_display: parsed.thread_display,
203                    source_file: None,
204                    format: LogFormat::CmtLog,
205                    file_path: file_path.to_string(),
206                    timezone_offset: Some(parsed.timezone_offset),
207                    error_code_spans: Vec::new(),
208                    ip_address: None,
209                    host_name: None,
210                    mac_address: None,
211                    result_code: None,
212                    gle_code: None,
213                    setup_phase: None,
214                    operation_name: None,
215                    http_method: None,
216                    uri_stem: None,
217                    uri_query: None,
218                    status_code: None,
219                    sub_status: None,
220                    time_taken_ms: None,
221                    client_ip: None,
222                    server_ip: None,
223                    user_agent: None,
224                    server_port: None,
225                    username: None,
226                    win32_status: None,
227                    query_name: None,
228                    query_type: None,
229                    response_code: None,
230                    dns_direction: None,
231                    dns_protocol: None,
232                    source_ip: None,
233                    dns_flags: None,
234                    dns_event_id: None,
235                    zone_name: None,
236                    entry_kind: None,
237                    whatif: None,
238                    section_name: None,
239                    section_color: None,
240                    iteration: None,
241                    tags: None,
242                };
243                (entry, comp.unwrap_or_default())
244            }
245            None => {
246                errors += 1;
247                let entry = LogEntry {
248                    id: id_counter,
249                    line_number: (i + 1) as u32,
250                    message: line.to_string(),
251                    component: None,
252                    timestamp: None,
253                    timestamp_display: None,
254                    severity: detect_severity_from_text(line),
255                    thread: None,
256                    thread_display: None,
257                    source_file: None,
258                    format: LogFormat::CmtLog,
259                    file_path: file_path.to_string(),
260                    timezone_offset: None,
261                    error_code_spans: Vec::new(),
262                    ip_address: None,
263                    host_name: None,
264                    mac_address: None,
265                    result_code: None,
266                    gle_code: None,
267                    setup_phase: None,
268                    operation_name: None,
269                    http_method: None,
270                    uri_stem: None,
271                    uri_query: None,
272                    status_code: None,
273                    sub_status: None,
274                    time_taken_ms: None,
275                    client_ip: None,
276                    server_ip: None,
277                    user_agent: None,
278                    server_port: None,
279                    username: None,
280                    win32_status: None,
281                    query_name: None,
282                    query_type: None,
283                    response_code: None,
284                    dns_direction: None,
285                    dns_protocol: None,
286                    source_ip: None,
287                    dns_flags: None,
288                    dns_event_id: None,
289                    zone_name: None,
290                    entry_kind: None,
291                    whatif: None,
292                    section_name: None,
293                    section_color: None,
294                    iteration: None,
295                    tags: None,
296                };
297                (entry, String::new())
298            }
299        };
300
301        let mut entry = base_entry;
302
303        // Classify by component name and extract CmtLog-specific attributes
304        match component_str.as_str() {
305            HEADER_COMPONENT => {
306                entry.entry_kind = Some(EntryKind::Header);
307            }
308            SECTION_COMPONENT => {
309                entry.entry_kind = Some(EntryKind::Section);
310                current_section_name = Some(entry.message.clone());
311                current_section_color = attrs.color.clone();
312                entry.section_name = Some(entry.message.clone());
313                entry.section_color = attrs.color;
314            }
315            ITERATION_COMPONENT => {
316                entry.entry_kind = Some(EntryKind::Iteration);
317                entry.iteration = attrs.iteration;
318                entry.section_color = attrs.color.or_else(|| current_section_color.clone());
319                entry.section_name = current_section_name.clone();
320            }
321            _ => {
322                entry.entry_kind = Some(EntryKind::Log);
323                entry.section_name = attrs.section.or_else(|| current_section_name.clone());
324                entry.section_color = current_section_color.clone();
325                entry.whatif = attrs.whatif.map(|v| v == "1");
326                entry.iteration = attrs.iteration;
327                entry.tags = attrs.tag.map(|t| {
328                    t.split(',')
329                        .map(|s| s.trim().to_string())
330                        .filter(|s| !s.is_empty())
331                        .collect()
332                });
333            }
334        }
335
336        entries.push(entry);
337        id_counter += 1;
338    }
339
340    (entries, errors)
341}
342
343#[cfg(test)]
344mod tests {
345    use super::*;
346    use crate::models::log_entry::Severity;
347
348    fn sample_lines() -> Vec<String> {
349        vec![
350            r##"<![LOG[Script started: Detect-WDAC.ps1 v2.1.0]LOG]!><time="10:30:00.000+000" date="04-13-2026" component="__HEADER__" context="" type="1" thread="0" file="" script="Detect-WDAC.ps1" version="2.1.0">"##.to_string(),
351            r##"<![LOG[Detection Phase]LOG]!><time="10:32:01.000+000" date="04-13-2026" component="__SECTION__" context="" type="1" thread="0" file="" color="#5b9aff">"##.to_string(),
352            r##"<![LOG[Scanning policy files]LOG]!><time="10:32:01.123+000" date="04-13-2026" component="Detect-WDAC" context="CONTOSO\admin" type="1" thread="1234" file="" section="detection" tag="phase:scan">"##.to_string(),
353        ]
354    }
355
356    #[test]
357    fn test_matches_cmtlog_record() {
358        assert!(matches_cmtlog_record(
359            r#"component="__HEADER__" context="" type="1""#
360        ));
361        assert!(matches_cmtlog_record(
362            r#"component="__SECTION__" context="" type="1""#
363        ));
364        assert!(matches_cmtlog_record(
365            r#"component="__ITERATION__" context="" type="1""#
366        ));
367        assert!(!matches_cmtlog_record(
368            r#"component="TestComp" context="" type="1""#
369        ));
370    }
371
372    #[test]
373    fn test_header_classification() {
374        let lines = sample_lines();
375        let line_refs: Vec<&str> = lines.iter().map(|s| s.as_str()).collect();
376        let (entries, _) = parse_lines(&line_refs, "test.cmtlog");
377        assert_eq!(entries[0].entry_kind, Some(EntryKind::Header));
378        assert_eq!(entries[0].format, LogFormat::CmtLog);
379    }
380
381    #[test]
382    fn test_section_classification() {
383        let lines = sample_lines();
384        let line_refs: Vec<&str> = lines.iter().map(|s| s.as_str()).collect();
385        let (entries, _) = parse_lines(&line_refs, "test.cmtlog");
386        assert_eq!(entries[1].entry_kind, Some(EntryKind::Section));
387        assert_eq!(entries[1].section_name.as_deref(), Some("Detection Phase"));
388        assert_eq!(entries[1].section_color.as_deref(), Some("#5b9aff"));
389    }
390
391    #[test]
392    fn test_log_inherits_section_context() {
393        let lines = sample_lines();
394        let line_refs: Vec<&str> = lines.iter().map(|s| s.as_str()).collect();
395        let (entries, _) = parse_lines(&line_refs, "test.cmtlog");
396        assert_eq!(entries[2].entry_kind, Some(EntryKind::Log));
397        // Explicit section attr overrides inherited section name
398        assert_eq!(entries[2].section_name.as_deref(), Some("detection"));
399        // Color is inherited from the current section
400        assert_eq!(entries[2].section_color.as_deref(), Some("#5b9aff"));
401        assert_eq!(entries[2].tags, Some(vec!["phase:scan".to_string()]));
402    }
403
404    #[test]
405    fn test_severity_mapping() {
406        let lines = [
407            r#"<![LOG[Policy validation failed]LOG]!><time="10:32:01.456+000" date="04-13-2026" component="Detect-WDAC" context="" type="3" thread="1234" file="">"#.to_string(),
408        ];
409        let line_refs: Vec<&str> = lines.iter().map(|s| s.as_str()).collect();
410        let (entries, _) = parse_lines(&line_refs, "test.cmtlog");
411        assert_eq!(entries[0].severity, Severity::Error);
412    }
413}