1use chrono::{
2 DateTime,
3 Utc,
4};
5use pipedash_plugin_api::{
6 LineMarker,
7 LogDialect,
8 ParsedLine,
9};
10
11const HEADER_LEN: usize = 32;
12const DATETIME_LEN: usize = 27;
13const SECTION_START: &[u8] = b"section_start:";
14const SECTION_END: &[u8] = b"section_end:";
15const CLEAR_LINE: &[u8] = b"\r\x1b[0K";
16
17pub struct GitlabLogDialect;
18
19struct Marker<'a> {
20 start: bool,
21 unix: i64,
22 name: &'a [u8],
23 collapsed: bool,
24 rest: &'a [u8],
25}
26
27impl LogDialect for GitlabLogDialect {
28 fn parse<'a>(&self, line: &'a [u8]) -> ParsedLine<'a> {
29 let (header_time, rest) = split_header(line);
30 let plain = ParsedLine {
31 timestamp: header_time,
32 marker: LineMarker::Plain,
33 body: rest,
34 };
35
36 let mut text = skip_csi(rest);
37 let mut started = None;
38 let mut ended = None;
39 let mut body = rest;
40 while let Some(marker) = parse_marker(text) {
41 if marker.start {
42 started = Some((marker.name, marker.unix, marker.collapsed));
43 } else {
44 ended = Some(marker.unix);
45 }
46 body = trim_carriage_returns(marker.rest);
47 text = skip_csi(marker.rest);
48 }
49
50 if let Some((name, unix, collapsed)) = started {
51 return ParsedLine {
52 timestamp: header_time.or_else(|| from_unix(unix)),
53 marker: LineMarker::SectionStart { open: !collapsed },
54 body: if is_blank(body) { name } else { body },
55 };
56 }
57 let Some(unix) = ended else {
58 return plain;
59 };
60 ParsedLine {
61 timestamp: header_time.or_else(|| from_unix(unix)),
62 marker: LineMarker::GroupEnd,
63 body: if is_blank(body) { &[] } else { body },
64 }
65 }
66}
67
68fn split_header(line: &[u8]) -> (Option<DateTime<Utc>>, &[u8]) {
69 let Some(header) = line.get(..HEADER_LEN) else {
70 return (None, line);
71 };
72 let well_formed = header[DATETIME_LEN - 1] == b'Z'
73 && header[DATETIME_LEN] == b' '
74 && header[DATETIME_LEN + 1..DATETIME_LEN + 3]
75 .iter()
76 .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(b))
77 && matches!(header[DATETIME_LEN + 3], b'E' | b'O')
78 && matches!(header[DATETIME_LEN + 4], b'+' | b' ');
79 let time = well_formed
80 .then(|| std::str::from_utf8(&header[..DATETIME_LEN]).ok())
81 .flatten()
82 .and_then(|text| DateTime::parse_from_rfc3339(text).ok());
83 match time {
84 Some(time) => (Some(time.with_timezone(&Utc)), &line[HEADER_LEN..]),
85 None => (None, line),
86 }
87}
88
89fn parse_marker(text: &[u8]) -> Option<Marker<'_>> {
90 let (start, rest) = match text.strip_prefix(SECTION_START) {
91 Some(rest) => (true, rest),
92 None => (false, text.strip_prefix(SECTION_END)?),
93 };
94 let digits = rest.iter().take_while(|b| b.is_ascii_digit()).count();
95 if digits == 0 || rest.get(digits) != Some(&b':') {
96 return None;
97 }
98 let unix = std::str::from_utf8(&rest[..digits]).ok()?.parse().ok()?;
99 let rest = &rest[digits + 1..];
100 let name_len = rest
101 .iter()
102 .take_while(|b| b.is_ascii_alphanumeric() || matches!(**b, b'_' | b'.' | b'-'))
103 .count();
104 if name_len == 0 {
105 return None;
106 }
107 let (name, mut rest) = rest.split_at(name_len);
108 let mut collapsed = false;
109 if rest.first() == Some(&b'[') {
110 let close = rest.iter().position(|b| *b == b']')?;
111 collapsed = rest[1..close]
112 .split(|b| *b == b',')
113 .any(|option| option.trim_ascii() == b"collapsed=true");
114 rest = &rest[close + 1..];
115 }
116 Some(Marker {
117 start,
118 unix,
119 name,
120 collapsed,
121 rest: rest.strip_prefix(CLEAR_LINE)?,
122 })
123}
124
125fn from_unix(seconds: i64) -> Option<DateTime<Utc>> {
126 DateTime::from_timestamp(seconds, 0)
127}
128
129fn skip_csi(mut text: &[u8]) -> &[u8] {
130 while let Some(rest) = text.strip_prefix(b"\x1b[") {
131 let Some(end) = rest.iter().position(|b| (0x40..=0x7e).contains(b)) else {
132 break;
133 };
134 text = &rest[end + 1..];
135 }
136 text
137}
138
139fn is_blank(mut text: &[u8]) -> bool {
140 loop {
141 text = skip_csi(text);
142 match text.split_first() {
143 None => return true,
144 Some((byte, rest)) if byte.is_ascii_whitespace() => text = rest,
145 Some(_) => return false,
146 }
147 }
148}
149
150fn trim_carriage_returns(mut text: &[u8]) -> &[u8] {
151 while let Some(rest) = text.strip_suffix(b"\r") {
152 text = rest;
153 }
154 text
155}
156
157#[cfg(test)]
158mod tests {
159 use super::*;
160
161 fn parse(line: &[u8]) -> ParsedLine<'_> {
162 GitlabLogDialect.parse(line)
163 }
164
165 fn millis(parsed: &ParsedLine<'_>) -> Option<i64> {
166 parsed.timestamp.map(|time| time.timestamp_millis())
167 }
168
169 #[test]
170 fn runner_timestamp_header_is_parsed_and_stripped() {
171 for (line, body) in [
172 (
173 b"2024-05-22T08:46:32.123456Z 00O Running with gitlab-runner 17.0.0".as_slice(),
174 b"Running with gitlab-runner 17.0.0".as_slice(),
175 ),
176 (b"2024-05-22T08:46:32.123456Z 01E boom", b"boom"),
177 (
178 b"2024-05-22T08:46:32.123456Z 00O+\x1b[0K\x1b[36;1mPreparing\x1b[0;m",
179 b"\x1b[0K\x1b[36;1mPreparing\x1b[0;m",
180 ),
181 (b"2024-05-22T08:46:32.123456Z 00O ", b""),
182 ] {
183 let parsed = parse(line);
184 assert_eq!(parsed.marker, LineMarker::Plain, "{line:?}");
185 assert_eq!(parsed.body, body, "{line:?}");
186 assert_eq!(millis(&parsed), Some(1_716_367_592_123), "{line:?}");
187 }
188 }
189
190 #[test]
191 fn lines_without_a_valid_header_keep_their_bytes() {
192 for line in [
193 b"plain output".as_slice(),
194 b"",
195 b"2024-05-22T08:46:32.123456Z 00O",
196 b"2024-05-22T08:46:32.123456Z 0zO text",
197 b"2024-05-22T08:46:32.123456Z 00X text",
198 b"2024-05-22T08:46:32.123456Z 00O?text",
199 b"2024-05-22T08:46:32.123Z 00O text 12345",
200 b"2024-13-22T08:46:32.123456Z 00O text",
201 ] {
202 let parsed = parse(line);
203 assert_eq!(parsed.timestamp, None, "{line:?}");
204 assert_eq!(parsed.marker, LineMarker::Plain, "{line:?}");
205 assert_eq!(parsed.body, line);
206 }
207 }
208
209 #[test]
210 fn section_start_opens_a_section_with_its_header() {
211 let parsed = parse(
212 b"section_start:1700000000:prepare_executor\r\x1b[0K\x1b[0K\x1b[36;1mPreparing the \"docker\" executor\x1b[0;m",
213 );
214 assert_eq!(parsed.marker, LineMarker::SectionStart { open: true });
215 assert_eq!(
216 parsed.body,
217 b"\x1b[0K\x1b[36;1mPreparing the \"docker\" executor\x1b[0;m"
218 );
219 assert_eq!(millis(&parsed), Some(1_700_000_000_000));
220 }
221
222 #[test]
223 fn collapsed_option_closes_the_section() {
224 let cases: [(&[u8], bool); 5] = [
225 (b"section_start:1:a[collapsed=true]\r\x1b[0KA", false),
226 (b"section_start:1:a[collapsed=false]\r\x1b[0KA", true),
227 (
228 b"section_start:1:a[hide_duration=true, collapsed=true]\r\x1b[0KA",
229 false,
230 ),
231 (
232 b"section_start:1:a[collapsed=true,hide_duration=true]\r\x1b[0KA",
233 false,
234 ),
235 (b"section_start:1:a[hide_duration=true]\r\x1b[0KA", true),
236 ];
237 for (line, open) in cases {
238 let parsed = parse(line);
239 assert_eq!(parsed.marker, LineMarker::SectionStart { open }, "{line:?}");
240 assert_eq!(parsed.body, b"A", "{line:?}");
241 }
242 }
243
244 #[test]
245 fn section_start_without_a_header_is_titled_by_its_name() {
246 let parsed =
247 parse(b"2024-05-22T08:46:32.123456Z 00O section_start:1:prepare_script\r\x1b[0K");
248 assert_eq!(parsed.marker, LineMarker::SectionStart { open: true });
249 assert_eq!(parsed.body, b"prepare_script");
250 assert_eq!(millis(&parsed), Some(1_716_367_592_123));
251 }
252
253 #[test]
254 fn section_end_closes_the_section() {
255 let parsed = parse(b"section_end:1700000007:step_script\r\x1b[0K");
256 assert_eq!(parsed.marker, LineMarker::GroupEnd);
257 assert_eq!(parsed.body, b"");
258 assert_eq!(millis(&parsed), Some(1_700_000_007_000));
259
260 let stamped = parse(b"2024-05-22T08:46:32.123456Z 00O \x1b[0Ksection_end:1:a\r\x1b[0K\r");
261 assert_eq!(stamped.marker, LineMarker::GroupEnd);
262 assert_eq!(millis(&stamped), Some(1_716_367_592_123));
263 }
264
265 #[test]
266 fn end_and_start_on_one_line_start_the_next_section() {
267 let parsed = parse(
268 b"section_end:1700000005:prepare_executor\r\x1b[0Ksection_start:1700000006:prepare_script\r\x1b[0KPreparing environment\r",
269 );
270 assert_eq!(parsed.marker, LineMarker::SectionStart { open: true });
271 assert_eq!(parsed.body, b"Preparing environment");
272 assert_eq!(millis(&parsed), Some(1_700_000_006_000));
273 }
274
275 #[test]
276 fn text_after_a_section_end_closes_the_section_and_stays_a_line() {
277 let parsed = parse(
278 b"section_end:1700000010:cleanup_file_variables\r\x1b[0K\x1b[32;1mJob succeeded\x1b[0;m",
279 );
280 assert_eq!(parsed.marker, LineMarker::GroupEnd);
281 assert_eq!(parsed.body, b"\x1b[32;1mJob succeeded\x1b[0;m");
282 }
283
284 #[test]
285 fn markers_follow_color_resets_but_not_output() {
286 let parsed = parse(b"\x1b[0;msection_start:1:build\r\x1b[0KBuild");
287 assert_eq!(parsed.marker, LineMarker::SectionStart { open: true });
288 assert_eq!(parsed.body, b"Build");
289
290 let output = b"echo section_start:1:build\r\x1b[0KBuild";
291 assert_eq!(parse(output).marker, LineMarker::Plain);
292 assert_eq!(parse(output).body, output);
293 }
294
295 #[test]
296 fn malformed_markers_are_output() {
297 for line in [
298 b"section_start:1:my_$ection\r\x1b[0Khello".as_slice(),
299 b"section_start:1:<a_tag>\r\x1b[0Khello",
300 b"section_start:1:name hello",
301 b"section_start:1:name\rhello",
302 b"section_start::name\r\x1b[0Khello",
303 b"section_start:1:\r\x1b[0Khello",
304 b"section_start:1:name[collapsed=true\r\x1b[0Khello",
305 b"section_end:x:name\r\x1b[0K",
306 ] {
307 let parsed = parse(line);
308 assert_eq!(parsed.marker, LineMarker::Plain, "{line:?}");
309 assert_eq!(parsed.body, line);
310 }
311 }
312}