1use std::io::{Cursor, Read, Write};
13
14use crate::{
15 context::{BuildEventContext, read_build_event_context},
16 error::MuninError,
17 field_flags::BuildEventArgsFieldFlags,
18 header::{BinlogHeader, open_binlog},
19 jsonlog::schema::{JsonlogEventBody, JsonlogFile, MUNIN_JSONLOG_VERSION},
20 nvl_table::{NameValueListTable, NameValuePair},
21 primitives::{read_7bit_count, read_7bit_int},
22 reader::{ArchiveEntry, BinlogEvent, dispatch_event},
23 record_kind::BinaryLogRecordKind,
24 string_table::StringTable,
25 writers::{WriteContext, write_7bit_int as w_7bit},
26};
27
28#[derive(Debug, Clone)]
37pub struct EventMeta {
38 pub record_kind: BinaryLogRecordKind,
40
41 pub byte_offset: u64,
43
44 pub payload_len: usize,
46
47 pub context: Option<BuildEventContext>,
49}
50
51#[derive(Debug, Clone)]
61struct IndexEntry {
62 meta: EventMeta,
63 payload: Vec<u8>,
64}
65
66#[derive(Debug)]
81pub struct BinlogIndex {
82 header: BinlogHeader,
83 strings: StringTable,
84 nvl_table: NameValueListTable,
85 archives: Vec<Vec<u8>>,
86 entries: Vec<IndexEntry>,
87}
88
89impl BinlogIndex {
90 pub fn open(reader: impl Read) -> Result<Self, MuninError> {
98 let (header, mut gz_reader) = open_binlog(reader)?;
99 let version = header.file_format_version;
100
101 let mut strings = StringTable::new();
102 let mut nvl_table = NameValueListTable::new();
103 let mut archives = Vec::new();
104 let mut entries = Vec::new();
105
106 let mut offset: u64 = 8; loop {
111 let kind_start = offset;
112
113 let (kind_raw, kind_bytes) = read_7bit_int_counted(&mut gz_reader)?;
115 offset += kind_bytes;
116
117 if kind_raw == BinaryLogRecordKind::EndOfFile as i32 {
118 break;
119 }
120
121 let (record_length, len_bytes) = read_7bit_int_counted(&mut gz_reader)?;
123 if record_length < 0 {
124 return Err(MuninError::InvalidFormat(format!(
125 "negative record length: {record_length}"
126 )));
127 }
128 let record_length = record_length as usize;
129 if record_length > crate::primitives::MAX_BINLOG_FIELD_LEN {
130 return Err(MuninError::InvalidFormat(format!(
131 "record length too large: {record_length} (max {})",
132 crate::primitives::MAX_BINLOG_FIELD_LEN
133 )));
134 }
135 offset += len_bytes;
136
137 let mut payload = vec![0u8; record_length];
139 gz_reader.read_exact(&mut payload)?;
140 offset += record_length as u64;
141
142 let kind = BinaryLogRecordKind::from_raw(kind_raw);
143
144 match kind {
145 Some(BinaryLogRecordKind::String) => {
146 let s = String::from_utf8(payload).map_err(|_| MuninError::InvalidUtf8)?;
147 strings.add(s);
148 }
149
150 Some(BinaryLogRecordKind::NameValueList) => {
151 let mut cursor = Cursor::new(&payload);
152 let count = read_7bit_count(&mut cursor, "name-value list count")?;
153 let mut pairs = Vec::with_capacity(count);
154 for _ in 0..count {
155 let key_index = read_7bit_int(&mut cursor)?;
156 let value_index = read_7bit_int(&mut cursor)?;
157 pairs.push(NameValuePair {
158 key_index,
159 value_index,
160 });
161 }
162 nvl_table.add(pairs);
163 }
164
165 Some(BinaryLogRecordKind::ProjectImportArchive) => {
166 archives.push(payload);
167 }
168
169 Some(record_kind) if !record_kind.is_auxiliary() => {
170 let context = extract_context(&payload, version);
173
174 let meta = EventMeta {
175 record_kind,
176 byte_offset: kind_start,
177 payload_len: record_length,
178 context,
179 };
180 entries.push(IndexEntry { meta, payload });
181 }
182
183 _ => {}
185 }
186 }
187
188 Ok(Self {
189 header,
190 strings,
191 nvl_table,
192 archives,
193 entries,
194 })
195 }
196
197 pub fn open_json(reader: impl Read) -> Result<Self, MuninError> {
207 let file: JsonlogFile = serde_json::from_reader(reader)
208 .map_err(|e| MuninError::InvalidFormat(format!("jsonlog parse: {e}")))?;
209 Self::from_jsonlog(file)
210 }
211
212 pub fn from_jsonlog(file: JsonlogFile) -> Result<Self, MuninError> {
214 if file.munin_jsonlog_version != MUNIN_JSONLOG_VERSION {
215 return Err(MuninError::InvalidFormat(format!(
216 "unsupported jsonlog version {} (expected {})",
217 file.munin_jsonlog_version, MUNIN_JSONLOG_VERSION
218 )));
219 }
220
221 let header: BinlogHeader = file.header.into();
222 let version = header.file_format_version;
223
224 let mut strings = StringTable::new();
226 for s in file.strings {
227 strings.add(s);
228 }
229
230 let mut nvl_table = NameValueListTable::new();
231 for entry in file.name_value_lists {
232 let pairs = entry
233 .into_iter()
234 .map(|pair| NameValuePair {
235 key_index: pair[0] as i32,
236 value_index: pair[1] as i32,
237 })
238 .collect();
239 nvl_table.add(pairs);
240 }
241
242 let archives: Vec<Vec<u8>> = file
243 .archives
244 .into_iter()
245 .map(|a| {
246 use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64};
247 BASE64
248 .decode(a.data_b64.as_bytes())
249 .map_err(|e| MuninError::InvalidFormat(format!("archive base64: {e}")))
250 })
251 .collect::<Result<Vec<_>, _>>()?;
252
253 let mut entries: Vec<IndexEntry> = Vec::with_capacity(file.events.len());
254 let mut ctx = WriteContext::with_tables(version, strings, nvl_table);
255 for ev in file.events {
256 let record_kind = BinaryLogRecordKind::from_name(&ev.kind).ok_or_else(|| {
257 MuninError::InvalidFormat(format!("unknown event kind: {}", ev.kind))
258 })?;
259 let payload = match ev.body {
260 JsonlogEventBody::PayloadB64(s) => {
261 use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64};
262 BASE64
263 .decode(s.as_bytes())
264 .map_err(|e| MuninError::InvalidFormat(format!("payload base64: {e}")))?
265 }
266 JsonlogEventBody::Decoded(value) => {
267 encode_decoded_event(record_kind, value, &mut ctx)?
268 }
269 };
270 let context = extract_context(&payload, version);
271 entries.push(IndexEntry {
272 meta: EventMeta {
273 record_kind,
274 byte_offset: ev.byte_offset,
275 payload_len: payload.len(),
276 context,
277 },
278 payload,
279 });
280 }
281 let strings = ctx.strings;
282 let nvl_table = ctx.nvl_table;
283
284 Ok(Self {
285 header,
286 strings,
287 nvl_table,
288 archives,
289 entries,
290 })
291 }
292
293 pub fn write_binlog<W: Write>(&self, writer: W) -> Result<(), MuninError> {
307 use flate2::{Compression, write::GzEncoder};
308
309 let mut gz = GzEncoder::new(writer, Compression::default());
310
311 gz.write_all(&self.header.file_format_version.to_le_bytes())?;
313 gz.write_all(&self.header.min_reader_version.to_le_bytes())?;
314
315 for s in self.strings.entries() {
317 let bytes = s.as_bytes();
318 w_7bit(&mut gz, BinaryLogRecordKind::String as i32)?;
319 w_7bit(&mut gz, bytes.len() as i32)?;
320 gz.write_all(bytes)?;
321 }
322
323 for list in self.nvl_table.entries() {
325 let mut buf = Vec::new();
327 w_7bit(&mut buf, list.len() as i32)?;
328 for pair in list {
329 w_7bit(&mut buf, pair.key_index)?;
330 w_7bit(&mut buf, pair.value_index)?;
331 }
332 w_7bit(&mut gz, BinaryLogRecordKind::NameValueList as i32)?;
333 w_7bit(&mut gz, buf.len() as i32)?;
334 gz.write_all(&buf)?;
335 }
336
337 for archive in &self.archives {
339 w_7bit(&mut gz, BinaryLogRecordKind::ProjectImportArchive as i32)?;
340 w_7bit(&mut gz, archive.len() as i32)?;
341 gz.write_all(archive)?;
342 }
343
344 for entry in &self.entries {
346 w_7bit(&mut gz, entry.meta.record_kind as i32)?;
347 w_7bit(&mut gz, entry.payload.len() as i32)?;
348 gz.write_all(&entry.payload)?;
349 }
350
351 w_7bit(&mut gz, BinaryLogRecordKind::EndOfFile as i32)?;
353
354 gz.finish()?;
355 Ok(())
356 }
357
358 pub fn header(&self) -> &BinlogHeader {
360 &self.header
361 }
362
363 pub fn strings(&self) -> &StringTable {
365 &self.strings
366 }
367
368 pub fn strings_mut(&mut self) -> &mut StringTable {
373 &mut self.strings
374 }
375
376 pub fn nvl_table(&self) -> &NameValueListTable {
378 &self.nvl_table
379 }
380
381 pub fn archives(&self) -> &[Vec<u8>] {
383 &self.archives
384 }
385
386 pub fn len(&self) -> usize {
388 self.entries.len()
389 }
390
391 pub fn is_empty(&self) -> bool {
393 self.entries.is_empty()
394 }
395
396 pub fn meta(&self, index: usize) -> Option<&EventMeta> {
400 self.entries.get(index).map(|e| &e.meta)
401 }
402
403 pub fn iter_meta(&self) -> impl Iterator<Item = (usize, &EventMeta)> {
405 self.entries.iter().enumerate().map(|(i, e)| (i, &e.meta))
406 }
407
408 pub fn payload_bytes(&self, index: usize) -> Option<&[u8]> {
416 self.entries.get(index).map(|e| e.payload.as_slice())
417 }
418
419 pub fn get(&self, index: usize) -> Result<Option<BinlogEvent>, MuninError> {
426 let entry = match self.entries.get(index) {
427 Some(e) => e,
428 None => return Ok(None),
429 };
430
431 let mut cursor = Cursor::new(&entry.payload);
432 let event = dispatch_event(
433 &mut cursor,
434 entry.meta.record_kind,
435 &self.strings,
436 &self.nvl_table,
437 self.header.file_format_version,
438 )?;
439 Ok(Some(event))
440 }
441
442 pub fn get_all(&self) -> Result<Vec<BinlogEvent>, MuninError> {
444 let mut events = Vec::with_capacity(self.entries.len());
445 for i in 0..self.entries.len() {
446 if let Some(event) = self.get(i)? {
447 events.push(event);
448 }
449 }
450 Ok(events)
451 }
452
453 pub fn indices_by_kind(&self, kind: BinaryLogRecordKind) -> Vec<usize> {
457 self.entries
458 .iter()
459 .enumerate()
460 .filter(|(_, e)| e.meta.record_kind == kind)
461 .map(|(i, _)| i)
462 .collect()
463 }
464
465 pub fn indices_by_project_context(&self, project_context_id: i32) -> Vec<usize> {
468 self.entries
469 .iter()
470 .enumerate()
471 .filter(|(_, e)| {
472 e.meta
473 .context
474 .is_some_and(|c| c.project_context_id == project_context_id)
475 })
476 .map(|(i, _)| i)
477 .collect()
478 }
479
480 pub fn indices_by_target_id(&self, target_id: i32) -> Vec<usize> {
482 self.entries
483 .iter()
484 .enumerate()
485 .filter(|(_, e)| e.meta.context.is_some_and(|c| c.target_id == target_id))
486 .map(|(i, _)| i)
487 .collect()
488 }
489
490 pub fn indices_by_task_id(&self, task_id: i32) -> Vec<usize> {
492 self.entries
493 .iter()
494 .enumerate()
495 .filter(|(_, e)| e.meta.context.is_some_and(|c| c.task_id == task_id))
496 .map(|(i, _)| i)
497 .collect()
498 }
499
500 pub fn query(
503 &self,
504 kind: Option<BinaryLogRecordKind>,
505 project_context_id: Option<i32>,
506 target_id: Option<i32>,
507 task_id: Option<i32>,
508 ) -> Vec<usize> {
509 self.entries
510 .iter()
511 .enumerate()
512 .filter(|(_, e)| {
513 if let Some(k) = kind
514 && e.meta.record_kind != k
515 {
516 return false;
517 }
518 if let Some(pid) = project_context_id
519 && e.meta.context.is_none_or(|c| c.project_context_id != pid)
520 {
521 return false;
522 }
523 if let Some(tid) = target_id
524 && e.meta.context.is_none_or(|c| c.target_id != tid)
525 {
526 return false;
527 }
528 if let Some(tsk) = task_id
529 && e.meta.context.is_none_or(|c| c.task_id != tsk)
530 {
531 return false;
532 }
533 true
534 })
535 .map(|(i, _)| i)
536 .collect()
537 }
538
539 pub fn extract_archives(&self) -> Result<Vec<ArchiveEntry>, MuninError> {
541 let mut entries = Vec::new();
543 for archive_bytes in &self.archives {
544 let cursor = Cursor::new(archive_bytes);
545 let mut archive = zip::ZipArchive::new(cursor).map_err(|e| {
546 MuninError::InvalidFormat(format!("invalid ProjectImportArchive zip: {e}"))
547 })?;
548 for i in 0..archive.len() {
549 let mut file = archive.by_index(i).map_err(|e| {
550 MuninError::InvalidFormat(format!("cannot read zip entry {i}: {e}"))
551 })?;
552 if file.is_dir() {
553 continue;
554 }
555 let path = file.name().to_string();
556 let mut contents = String::new();
557 if file.read_to_string(&mut contents).is_ok() {
558 entries.push(ArchiveEntry { path, contents });
559 }
560 }
562 }
563 Ok(entries)
564 }
565}
566
567fn read_7bit_int_counted(reader: &mut impl Read) -> Result<(i32, u64), MuninError> {
574 let mut result: u32 = 0;
575 let mut shift: u32 = 0;
576 let mut buf = [0u8; 1];
577 let mut count: u64 = 0;
578
579 for _ in 0..5 {
580 reader.read_exact(&mut buf)?;
581 count += 1;
582 let byte = buf[0];
583 result |= ((byte & 0x7F) as u32) << shift;
584 if byte & 0x80 == 0 {
585 return Ok((result as i32, count));
586 }
587 shift += 7;
588 }
589
590 Err(MuninError::OverlongVarInt)
591}
592
593fn extract_context(payload: &[u8], file_format_version: i32) -> Option<BuildEventContext> {
599 let mut cursor = Cursor::new(payload);
600
601 let flags_raw = read_7bit_int(&mut cursor).ok()? as u32;
603 let flags = BuildEventArgsFieldFlags::from_raw(flags_raw);
604
605 if flags.contains(BuildEventArgsFieldFlags::MESSAGE) {
607 let _ = read_7bit_int(&mut cursor).ok()?;
608 }
609
610 if flags.contains(BuildEventArgsFieldFlags::BUILD_EVENT_CONTEXT) {
612 read_build_event_context(&mut cursor, file_format_version).ok()
613 } else {
614 None
615 }
616}
617
618#[cfg(test)]
619mod tests;
620
621fn encode_decoded_event(
629 kind: BinaryLogRecordKind,
630 value: serde_json::Value,
631 ctx: &mut WriteContext,
632) -> Result<Vec<u8>, MuninError> {
633 use crate::events as ev;
634 let mut buf = Vec::new();
635 macro_rules! enc {
636 ($Ty:ty, $write:path) => {{
637 let parsed: $Ty = serde_json::from_value(value)
638 .map_err(|e| MuninError::InvalidFormat(format!("decoded event: {e}")))?;
639 $write(&mut buf, ctx, &parsed)
640 .map_err(|e| MuninError::InvalidFormat(format!("encode event: {e}")))?;
641 }};
642 }
643 match kind {
644 BinaryLogRecordKind::BuildStarted => enc!(ev::BuildStartedEvent, ev::write_build_started),
645 BinaryLogRecordKind::BuildFinished => {
646 enc!(ev::BuildFinishedEvent, ev::write_build_finished)
647 }
648 BinaryLogRecordKind::ProjectStarted => {
649 enc!(ev::ProjectStartedEvent, ev::write_project_started)
650 }
651 BinaryLogRecordKind::ProjectFinished => {
652 enc!(ev::ProjectFinishedEvent, ev::write_project_finished)
653 }
654 BinaryLogRecordKind::TargetStarted => {
655 enc!(ev::TargetStartedEvent, ev::write_target_started)
656 }
657 BinaryLogRecordKind::TargetFinished => {
658 enc!(ev::TargetFinishedEvent, ev::write_target_finished)
659 }
660 BinaryLogRecordKind::TargetSkipped => {
661 enc!(ev::TargetSkippedEvent, ev::write_target_skipped)
662 }
663 BinaryLogRecordKind::TaskStarted => enc!(ev::TaskStartedEvent, ev::write_task_started),
664 BinaryLogRecordKind::TaskFinished => enc!(ev::TaskFinishedEvent, ev::write_task_finished),
665 BinaryLogRecordKind::TaskCommandLine => {
666 enc!(ev::TaskCommandLineEvent, ev::write_task_command_line)
667 }
668 BinaryLogRecordKind::TaskParameter => {
669 enc!(ev::TaskParameterEvent, ev::write_task_parameter)
670 }
671 BinaryLogRecordKind::Error => enc!(ev::BuildErrorEvent, ev::write_build_error),
672 BinaryLogRecordKind::Warning => enc!(ev::BuildWarningEvent, ev::write_build_warning),
673 BinaryLogRecordKind::Message => enc!(ev::BuildMessageEvent, ev::write_build_message),
674 BinaryLogRecordKind::CriticalBuildMessage => {
675 enc!(
676 ev::CriticalBuildMessageEvent,
677 ev::write_critical_build_message
678 )
679 }
680 BinaryLogRecordKind::ProjectEvaluationStarted => enc!(
681 ev::ProjectEvaluationStartedEvent,
682 ev::write_project_evaluation_started
683 ),
684 BinaryLogRecordKind::ProjectEvaluationFinished => enc!(
685 ev::ProjectEvaluationFinishedEvent,
686 ev::write_project_evaluation_finished
687 ),
688 BinaryLogRecordKind::PropertyReassignment => enc!(
689 ev::PropertyReassignmentEvent,
690 ev::write_property_reassignment
691 ),
692 BinaryLogRecordKind::UninitializedPropertyRead => enc!(
693 ev::UninitializedPropertyReadEvent,
694 ev::write_uninitialized_property_read
695 ),
696 BinaryLogRecordKind::PropertyInitialValueSet => enc!(
697 ev::PropertyInitialValueSetEvent,
698 ev::write_property_initial_value_set
699 ),
700 BinaryLogRecordKind::EnvironmentVariableRead => enc!(
701 ev::EnvironmentVariableReadEvent,
702 ev::write_environment_variable_read
703 ),
704 BinaryLogRecordKind::ResponseFileUsed => {
705 enc!(ev::ResponseFileUsedEvent, ev::write_response_file_used)
706 }
707 BinaryLogRecordKind::AssemblyLoad => enc!(ev::AssemblyLoadEvent, ev::write_assembly_load),
708 BinaryLogRecordKind::ProjectImported => {
709 enc!(ev::ProjectImportedEvent, ev::write_project_imported)
710 }
711 BinaryLogRecordKind::BuildCheckMessage => {
712 enc!(ev::BuildCheckMessageEvent, ev::write_build_message)
713 }
714 BinaryLogRecordKind::BuildCheckWarning => {
715 enc!(ev::BuildCheckWarningEvent, ev::write_build_warning)
716 }
717 BinaryLogRecordKind::BuildCheckError => {
718 enc!(ev::BuildCheckErrorEvent, ev::write_build_error)
719 }
720 BinaryLogRecordKind::BuildCheckTracing => {
721 enc!(ev::BuildCheckTracingEvent, ev::write_build_check_tracing)
722 }
723 BinaryLogRecordKind::BuildCheckAcquisition => enc!(
724 ev::BuildCheckAcquisitionEvent,
725 ev::write_build_check_acquisition
726 ),
727 BinaryLogRecordKind::BuildSubmissionStarted => enc!(
728 ev::BuildSubmissionStartedEvent,
729 ev::write_build_submission_started
730 ),
731 BinaryLogRecordKind::BuildCanceled => {
732 enc!(ev::BuildCanceledEvent, ev::write_build_canceled)
733 }
734 BinaryLogRecordKind::EndOfFile
735 | BinaryLogRecordKind::String
736 | BinaryLogRecordKind::NameValueList
737 | BinaryLogRecordKind::ProjectImportArchive => {
738 return Err(MuninError::InvalidFormat(format!(
739 "cannot encode auxiliary record kind: {:?}",
740 kind
741 )));
742 }
743 }
744 Ok(buf)
745}