1use serde::Serialize;
16use std::collections::HashMap;
17use std::io::{Read, Seek};
18
19use crate::innodb::log::{
20 compute_record_lsn, parse_mlog_records, LogBlockHeader, LogFile, LOG_FILE_HDR_BLOCKS,
21};
22use crate::innodb::page::FilHeader;
23use crate::innodb::page_types::PageType;
24use crate::innodb::undo::{parse_undo_records, UndoRecordType};
25use crate::IdbError;
26
27#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
31pub enum TimelineSource {
32 RedoLog,
33 UndoLog,
34 Binlog,
35}
36
37impl std::fmt::Display for TimelineSource {
38 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
39 match self {
40 TimelineSource::RedoLog => write!(f, "REDO"),
41 TimelineSource::UndoLog => write!(f, "UNDO"),
42 TimelineSource::Binlog => write!(f, "BINLOG"),
43 }
44 }
45}
46
47#[derive(Debug, Clone, Serialize)]
49#[serde(tag = "type")]
50pub enum TimelineAction {
51 Redo { mlog_type: String, single_rec: bool },
53 Undo {
55 record_type: String,
56 trx_id: u64,
57 undo_no: u64,
58 table_id: u64,
59 },
60 Binlog {
62 event_type: String,
63 #[serde(skip_serializing_if = "Option::is_none")]
64 database: Option<String>,
65 #[serde(skip_serializing_if = "Option::is_none")]
66 table: Option<String>,
67 #[serde(skip_serializing_if = "Option::is_none")]
68 xid: Option<u64>,
69 #[serde(skip_serializing_if = "Option::is_none")]
71 pk_values: Option<Vec<String>>,
72 },
73}
74
75#[derive(Debug, Clone, Serialize)]
77pub struct TimelineEntry {
78 pub seq: u64,
80 pub source: TimelineSource,
82 #[serde(skip_serializing_if = "Option::is_none")]
84 pub lsn: Option<u64>,
85 #[serde(skip_serializing_if = "Option::is_none")]
87 pub timestamp: Option<u32>,
88 #[serde(skip_serializing_if = "Option::is_none")]
90 pub space_id: Option<u32>,
91 #[serde(skip_serializing_if = "Option::is_none")]
93 pub page_no: Option<u32>,
94 pub action: TimelineAction,
96}
97
98#[derive(Debug, Clone, Serialize)]
100pub struct PageTimelineSummary {
101 pub space_id: u32,
102 pub page_no: u32,
103 pub redo_entries: usize,
104 pub undo_entries: usize,
105 pub binlog_entries: usize,
106 #[serde(skip_serializing_if = "Option::is_none")]
107 pub first_lsn: Option<u64>,
108 #[serde(skip_serializing_if = "Option::is_none")]
109 pub last_lsn: Option<u64>,
110}
111
112#[derive(Debug, Clone, Serialize)]
114pub struct TimelineReport {
115 pub redo_count: usize,
117 pub undo_count: usize,
118 pub binlog_count: usize,
119 pub correlated_count: usize,
121 pub entries: Vec<TimelineEntry>,
123 #[serde(skip_serializing_if = "Vec::is_empty")]
125 pub page_summaries: Vec<PageTimelineSummary>,
126}
127
128pub fn extract_redo_timeline(log: &mut LogFile) -> Result<Vec<TimelineEntry>, IdbError> {
135 let header = log.read_header()?;
136 let data_blocks = log.data_block_count();
137 let mut entries = Vec::new();
138
139 for i in 0..data_blocks {
140 let block_idx = LOG_FILE_HDR_BLOCKS + i;
141 let block_data = log.read_block(block_idx)?;
142 let hdr = match LogBlockHeader::parse(&block_data) {
143 Some(h) if h.has_data() => h,
144 _ => continue,
145 };
146
147 let records = parse_mlog_records(&block_data, &hdr);
148 for rec in records {
149 let lsn = compute_record_lsn(header.start_lsn, i, rec.block_offset);
150 entries.push(TimelineEntry {
151 seq: 0, source: TimelineSource::RedoLog,
153 lsn: Some(lsn),
154 timestamp: None,
155 space_id: rec.space_id,
156 page_no: rec.page_no,
157 action: TimelineAction::Redo {
158 mlog_type: rec.record_type.to_string(),
159 single_rec: rec.single_rec,
160 },
161 });
162 }
163 }
164
165 Ok(entries)
166}
167
168pub fn extract_undo_timeline(
175 ts: &mut crate::innodb::tablespace::Tablespace,
176) -> Result<Vec<TimelineEntry>, IdbError> {
177 let page_count = ts.page_count();
178 let mut entries = Vec::new();
179
180 for pn in 0..page_count {
181 let page_data = ts.read_page(pn)?;
182 let fil = match FilHeader::parse(&page_data) {
183 Some(h) => h,
184 None => continue,
185 };
186
187 if fil.page_type != PageType::UndoLog {
189 continue;
190 }
191
192 let records = parse_undo_records(&page_data);
193 for rec in records {
194 let record_type_str = match rec.record_type {
195 UndoRecordType::InsertRec => "INSERT",
196 UndoRecordType::UpdExistRec => "UPDATE",
197 UndoRecordType::UpdDelRec => "UPDATE_DELETE",
198 UndoRecordType::DelMarkRec => "DELETE_MARK",
199 UndoRecordType::Unknown(_) => "UNKNOWN",
200 };
201
202 entries.push(TimelineEntry {
203 seq: 0,
204 source: TimelineSource::UndoLog,
205 lsn: Some(fil.lsn),
206 timestamp: None,
207 space_id: None,
208 page_no: Some(fil.page_number),
209 action: TimelineAction::Undo {
210 record_type: record_type_str.to_string(),
211 trx_id: rec.trx_id.unwrap_or(0),
212 undo_no: rec.undo_no,
213 table_id: rec.table_id,
214 },
215 });
216 }
217 }
218
219 Ok(entries)
220}
221
222pub fn extract_binlog_timeline<R: Read + Seek>(reader: R) -> Result<Vec<TimelineEntry>, IdbError> {
230 let analysis = crate::binlog::events::analyze_binlog(reader)?;
231
232 let mut table_map_idx: usize = 0;
237 let mut current_db: Option<String> = None;
238 let mut current_table: Option<String> = None;
239
240 let mut entries = Vec::new();
241
242 for ev in &analysis.events {
243 match ev.type_code {
244 19 => {
246 if table_map_idx < analysis.table_maps.len() {
247 let tme = &analysis.table_maps[table_map_idx];
248 current_db = Some(tme.database_name.clone());
249 current_table = Some(tme.table_name.clone());
250 table_map_idx += 1;
251 }
252 }
253 30..=32 => {
255 entries.push(TimelineEntry {
256 seq: 0,
257 source: TimelineSource::Binlog,
258 lsn: None,
259 timestamp: Some(ev.timestamp),
260 space_id: None,
261 page_no: None,
262 action: TimelineAction::Binlog {
263 event_type: ev.event_type.clone(),
264 database: current_db.clone(),
265 table: current_table.clone(),
266 xid: None,
267 pk_values: None,
268 },
269 });
270 }
271 2 => {
273 entries.push(TimelineEntry {
274 seq: 0,
275 source: TimelineSource::Binlog,
276 lsn: None,
277 timestamp: Some(ev.timestamp),
278 space_id: None,
279 page_no: None,
280 action: TimelineAction::Binlog {
281 event_type: ev.event_type.clone(),
282 database: None,
283 table: None,
284 xid: None,
285 pk_values: None,
286 },
287 });
288 }
289 _ => {}
290 }
291 }
292
293 Ok(entries)
294}
295
296pub struct BinlogExtractionResult {
298 pub entries: Vec<TimelineEntry>,
300 pub table_maps: HashMap<u64, crate::binlog::events::TableMapEvent>,
302 pub row_data: HashMap<usize, Vec<u8>>,
304}
305
306pub fn extract_binlog_timeline_enriched<R: Read + Seek>(
312 reader: R,
313) -> Result<BinlogExtractionResult, IdbError> {
314 use crate::binlog::constants::COMMON_HEADER_SIZE;
315 use crate::binlog::events::{RowsEvent, TableMapEvent};
316 use crate::binlog::header::{validate_binlog_magic, BinlogEventHeader};
317 use std::io::SeekFrom;
318
319 let mut reader = reader;
320
321 let mut magic = [0u8; 4];
323 reader
324 .read_exact(&mut magic)
325 .map_err(|e| IdbError::Io(format!("Failed to read binlog magic: {e}")))?;
326
327 if !validate_binlog_magic(&magic) {
328 return Err(IdbError::Parse(
329 "Not a valid MySQL binary log file (bad magic)".to_string(),
330 ));
331 }
332
333 let file_size = reader
334 .seek(SeekFrom::End(0))
335 .map_err(|e| IdbError::Io(format!("Failed to seek: {e}")))?;
336 reader
337 .seek(SeekFrom::Start(4))
338 .map_err(|e| IdbError::Io(format!("Failed to seek: {e}")))?;
339
340 let mut entries = Vec::new();
341 let mut table_maps: HashMap<u64, TableMapEvent> = HashMap::new();
342 let mut row_data_map: HashMap<usize, Vec<u8>> = HashMap::new();
343
344 let mut current_db: Option<String> = None;
345 let mut current_table: Option<String> = None;
346
347 let mut position = 4u64;
348 let mut header_buf = vec![0u8; COMMON_HEADER_SIZE];
349
350 while position + COMMON_HEADER_SIZE as u64 <= file_size {
351 if reader.read_exact(&mut header_buf).is_err() {
352 break;
353 }
354
355 let hdr = match BinlogEventHeader::parse(&header_buf) {
356 Some(h) => h,
357 None => break,
358 };
359
360 if hdr.event_length < COMMON_HEADER_SIZE as u32 {
361 break;
362 }
363
364 let data_len = hdr.event_length as usize - COMMON_HEADER_SIZE;
365 let mut event_data = vec![0u8; data_len];
366 if reader.read_exact(&mut event_data).is_err() {
367 break;
368 }
369
370 match hdr.type_code {
371 19 => {
373 if let Some(tme) = TableMapEvent::parse(&event_data) {
374 current_db = Some(tme.database_name.clone());
375 current_table = Some(tme.table_name.clone());
376 table_maps.insert(tme.table_id, tme);
377 }
378 }
379 30..=32 => {
381 let entry_idx = entries.len();
382 entries.push(TimelineEntry {
383 seq: 0,
384 source: TimelineSource::Binlog,
385 lsn: None,
386 timestamp: Some(hdr.timestamp),
387 space_id: None,
388 page_no: None,
389 action: TimelineAction::Binlog {
390 event_type: crate::binlog::event::BinlogEventType::from_u8(hdr.type_code)
391 .name()
392 .to_string(),
393 database: current_db.clone(),
394 table: current_table.clone(),
395 xid: None,
396 pk_values: None,
397 },
398 });
399
400 if let Some(rows_ev) = RowsEvent::parse(&event_data, hdr.type_code) {
402 if !rows_ev.row_data.is_empty() {
403 row_data_map.insert(entry_idx, rows_ev.row_data);
404 }
405 }
406 }
407 2 => {
409 entries.push(TimelineEntry {
410 seq: 0,
411 source: TimelineSource::Binlog,
412 lsn: None,
413 timestamp: Some(hdr.timestamp),
414 space_id: None,
415 page_no: None,
416 action: TimelineAction::Binlog {
417 event_type: "QUERY".to_string(),
418 database: None,
419 table: None,
420 xid: None,
421 pk_values: None,
422 },
423 });
424 }
425 _ => {}
426 }
427
428 position = if hdr.next_position > 0 {
429 hdr.next_position as u64
430 } else {
431 position + hdr.event_length as u64
432 };
433
434 if reader.seek(SeekFrom::Start(position)).is_err() {
435 break;
436 }
437 }
438
439 Ok(BinlogExtractionResult {
440 entries,
441 table_maps,
442 row_data: row_data_map,
443 })
444}
445
446pub fn correlate_binlog_pages(
456 entries: &mut [TimelineEntry],
457 ts: &mut crate::innodb::tablespace::Tablespace,
458 table_maps: &HashMap<u64, crate::binlog::events::TableMapEvent>,
459 row_data_map: &HashMap<usize, Vec<u8>>,
460) -> Result<usize, IdbError> {
461 use crate::binlog::correlate::{
462 build_column_meta, convert_pk_values, extract_ddl_column_names,
463 };
464 use crate::binlog::row_image::extract_pk_from_row_image;
465 use crate::innodb::btree::{extract_clustered_index_info, search_btree};
466
467 let (root_page_no, index_id, pk_columns) = match extract_clustered_index_info(ts) {
469 Some(info) => info,
470 None => return Ok(0), };
472
473 let ddl_column_names = extract_ddl_column_names(ts).unwrap_or_default();
475
476 let page0 = ts.read_page(0)?;
478 let space_id = FilHeader::parse(&page0).map(|h| h.space_id);
479 let page_size = ts.page_size();
480
481 let mut correlated = 0usize;
482
483 for (entry_idx, entry) in entries.iter_mut().enumerate() {
484 let row_data = match row_data_map.get(&entry_idx) {
486 Some(data) if !data.is_empty() => data,
487 _ => continue,
488 };
489
490 let (db_name, tbl_name) = match &entry.action {
493 TimelineAction::Binlog {
494 database: Some(db),
495 table: Some(tbl),
496 ..
497 } => (db.clone(), tbl.clone()),
498 _ => continue,
499 };
500
501 let tme = match table_maps
503 .values()
504 .find(|t| t.database_name == db_name && t.table_name == tbl_name)
505 {
506 Some(t) => t,
507 None => continue,
508 };
509
510 let columns = build_column_meta(tme, &pk_columns, &ddl_column_names);
512
513 let pk_values = match extract_pk_from_row_image(row_data, &columns) {
515 Some(pks) => pks,
516 None => continue,
517 };
518
519 let search_key = convert_pk_values(&pk_values);
521
522 match search_btree(
524 ts,
525 root_page_no,
526 index_id,
527 &pk_columns,
528 &search_key,
529 page_size,
530 ) {
531 Ok(result) => {
532 entry.page_no = Some(result.leaf_page_no);
533 entry.space_id = space_id;
534
535 let pk_strs: Vec<String> = pk_values.iter().map(|v| v.to_string()).collect();
537 if let TimelineAction::Binlog {
538 ref mut pk_values, ..
539 } = entry.action
540 {
541 *pk_values = Some(pk_strs);
542 }
543
544 correlated += 1;
545 }
546 Err(_) => {
547 continue;
549 }
550 }
551 }
552
553 Ok(correlated)
554}
555
556pub fn merge_timeline(
567 mut redo: Vec<TimelineEntry>,
568 mut undo: Vec<TimelineEntry>,
569 mut binlog: Vec<TimelineEntry>,
570) -> TimelineReport {
571 let redo_count = redo.len();
572 let undo_count = undo.len();
573 let binlog_count = binlog.len();
574
575 let mut all = Vec::with_capacity(redo_count + undo_count + binlog_count);
576 all.append(&mut redo);
577 all.append(&mut undo);
578 all.append(&mut binlog);
579
580 all.sort_by(|a, b| {
582 let lsn_cmp = a.lsn.unwrap_or(u64::MAX).cmp(&b.lsn.unwrap_or(u64::MAX));
583 if lsn_cmp != std::cmp::Ordering::Equal {
584 return lsn_cmp;
585 }
586 a.timestamp
587 .unwrap_or(u32::MAX)
588 .cmp(&b.timestamp.unwrap_or(u32::MAX))
589 });
590
591 for (i, entry) in all.iter_mut().enumerate() {
593 entry.seq = (i + 1) as u64;
594 }
595
596 let mut page_agg: HashMap<(u32, u32), PageTimelineSummary> = HashMap::new();
598 for entry in &all {
599 if let (Some(sid), Some(pno)) = (entry.space_id, entry.page_no) {
600 let summary = page_agg.entry((sid, pno)).or_insert(PageTimelineSummary {
601 space_id: sid,
602 page_no: pno,
603 redo_entries: 0,
604 undo_entries: 0,
605 binlog_entries: 0,
606 first_lsn: None,
607 last_lsn: None,
608 });
609 match entry.source {
610 TimelineSource::RedoLog => summary.redo_entries += 1,
611 TimelineSource::UndoLog => summary.undo_entries += 1,
612 TimelineSource::Binlog => summary.binlog_entries += 1,
613 }
614 if let Some(lsn) = entry.lsn {
615 summary.first_lsn = Some(summary.first_lsn.map_or(lsn, |v: u64| v.min(lsn)));
616 summary.last_lsn = Some(summary.last_lsn.map_or(lsn, |v: u64| v.max(lsn)));
617 }
618 }
619 }
620
621 let correlated_count = page_agg
623 .values()
624 .filter(|s| {
625 let sources = [s.redo_entries > 0, s.undo_entries > 0, s.binlog_entries > 0];
626 sources.iter().filter(|&&v| v).count() >= 2
627 })
628 .count();
629
630 let mut page_summaries: Vec<PageTimelineSummary> = page_agg.into_values().collect();
631 page_summaries.sort_by_key(|s| (s.space_id, s.page_no));
632
633 TimelineReport {
634 redo_count,
635 undo_count,
636 binlog_count,
637 correlated_count,
638 entries: all,
639 page_summaries,
640 }
641}
642
643#[cfg(not(target_arch = "wasm32"))]
651pub fn build_space_table_map(datadir: &str) -> Result<HashMap<u32, String>, IdbError> {
652 use std::path::Path;
653
654 use crate::innodb::tablespace::Tablespace;
655 use crate::util::fs::find_tablespace_files;
656
657 let files = find_tablespace_files(Path::new(datadir), &["ibd"], None)?;
658 let mut map = HashMap::new();
659
660 for path in files {
661 let path_str = path.to_string_lossy().to_string();
662 if let Ok(mut ts) = Tablespace::open(&path_str) {
663 let space_id = ts
665 .read_page(0)
666 .ok()
667 .and_then(|p| FilHeader::parse(&p))
668 .map(|h| h.space_id);
669
670 if let Some(sid) = space_id {
671 let p = Path::new(&path_str);
673 let table = p.file_stem().and_then(|s| s.to_str()).unwrap_or("unknown");
674 let db = p
675 .parent()
676 .and_then(|d| d.file_name())
677 .and_then(|s| s.to_str())
678 .unwrap_or("");
679 let full = if db.is_empty() {
680 table.to_string()
681 } else {
682 format!("{}.{}", db, table)
683 };
684 map.insert(sid, full);
685 }
686 }
687 }
688
689 Ok(map)
690}
691
692#[cfg(test)]
693mod tests {
694 use super::*;
695
696 #[test]
697 fn test_merge_empty() {
698 let report = merge_timeline(vec![], vec![], vec![]);
699 assert_eq!(report.redo_count, 0);
700 assert_eq!(report.undo_count, 0);
701 assert_eq!(report.binlog_count, 0);
702 assert_eq!(report.correlated_count, 0);
703 assert!(report.entries.is_empty());
704 assert!(report.page_summaries.is_empty());
705 }
706
707 #[test]
708 fn test_merge_sort_by_lsn() {
709 let redo = vec![
710 TimelineEntry {
711 seq: 0,
712 source: TimelineSource::RedoLog,
713 lsn: Some(200),
714 timestamp: None,
715 space_id: Some(5),
716 page_no: Some(3),
717 action: TimelineAction::Redo {
718 mlog_type: "MLOG_REC_INSERT".to_string(),
719 single_rec: true,
720 },
721 },
722 TimelineEntry {
723 seq: 0,
724 source: TimelineSource::RedoLog,
725 lsn: Some(100),
726 timestamp: None,
727 space_id: Some(5),
728 page_no: Some(3),
729 action: TimelineAction::Redo {
730 mlog_type: "MLOG_REC_DELETE".to_string(),
731 single_rec: false,
732 },
733 },
734 ];
735
736 let undo = vec![TimelineEntry {
737 seq: 0,
738 source: TimelineSource::UndoLog,
739 lsn: Some(150),
740 timestamp: None,
741 space_id: None,
742 page_no: Some(7),
743 action: TimelineAction::Undo {
744 record_type: "INSERT".to_string(),
745 trx_id: 42,
746 undo_no: 1,
747 table_id: 10,
748 },
749 }];
750
751 let report = merge_timeline(redo, undo, vec![]);
752 assert_eq!(report.redo_count, 2);
753 assert_eq!(report.undo_count, 1);
754 assert_eq!(report.entries.len(), 3);
755
756 assert_eq!(report.entries[0].lsn, Some(100));
758 assert_eq!(report.entries[0].seq, 1);
759 assert_eq!(report.entries[1].lsn, Some(150));
760 assert_eq!(report.entries[1].seq, 2);
761 assert_eq!(report.entries[2].lsn, Some(200));
762 assert_eq!(report.entries[2].seq, 3);
763 }
764
765 #[test]
766 fn test_merge_page_summaries() {
767 let redo = vec![
768 TimelineEntry {
769 seq: 0,
770 source: TimelineSource::RedoLog,
771 lsn: Some(100),
772 timestamp: None,
773 space_id: Some(5),
774 page_no: Some(3),
775 action: TimelineAction::Redo {
776 mlog_type: "MLOG_REC_INSERT".to_string(),
777 single_rec: true,
778 },
779 },
780 TimelineEntry {
781 seq: 0,
782 source: TimelineSource::RedoLog,
783 lsn: Some(200),
784 timestamp: None,
785 space_id: Some(5),
786 page_no: Some(3),
787 action: TimelineAction::Redo {
788 mlog_type: "MLOG_REC_DELETE".to_string(),
789 single_rec: true,
790 },
791 },
792 ];
793
794 let report = merge_timeline(redo, vec![], vec![]);
795 assert_eq!(report.page_summaries.len(), 1);
796 let ps = &report.page_summaries[0];
797 assert_eq!(ps.space_id, 5);
798 assert_eq!(ps.page_no, 3);
799 assert_eq!(ps.redo_entries, 2);
800 assert_eq!(ps.first_lsn, Some(100));
801 assert_eq!(ps.last_lsn, Some(200));
802 }
803
804 #[test]
805 fn test_merge_correlated_count() {
806 let redo = vec![TimelineEntry {
807 seq: 0,
808 source: TimelineSource::RedoLog,
809 lsn: Some(100),
810 timestamp: None,
811 space_id: Some(5),
812 page_no: Some(3),
813 action: TimelineAction::Redo {
814 mlog_type: "MLOG_REC_INSERT".to_string(),
815 single_rec: true,
816 },
817 }];
818 let undo = vec![TimelineEntry {
819 seq: 0,
820 source: TimelineSource::UndoLog,
821 lsn: Some(150),
822 timestamp: None,
823 space_id: Some(5),
824 page_no: Some(3),
825 action: TimelineAction::Undo {
826 record_type: "INSERT".to_string(),
827 trx_id: 1,
828 undo_no: 1,
829 table_id: 1,
830 },
831 }];
832
833 let report = merge_timeline(redo, undo, vec![]);
834 assert_eq!(report.correlated_count, 1);
835 }
836
837 #[test]
838 fn test_binlog_entries_sort_after_lsn() {
839 let redo = vec![TimelineEntry {
840 seq: 0,
841 source: TimelineSource::RedoLog,
842 lsn: Some(100),
843 timestamp: None,
844 space_id: Some(5),
845 page_no: Some(3),
846 action: TimelineAction::Redo {
847 mlog_type: "MLOG_REC_INSERT".to_string(),
848 single_rec: true,
849 },
850 }];
851 let binlog = vec![TimelineEntry {
852 seq: 0,
853 source: TimelineSource::Binlog,
854 lsn: None,
855 timestamp: Some(1700000000),
856 space_id: None,
857 page_no: None,
858 action: TimelineAction::Binlog {
859 event_type: "WRITE_ROWS_EVENT_V2".to_string(),
860 database: Some("test".to_string()),
861 table: Some("users".to_string()),
862 xid: None,
863 pk_values: None,
864 },
865 }];
866
867 let report = merge_timeline(redo, vec![], binlog);
868 assert_eq!(report.entries.len(), 2);
869 assert_eq!(report.entries[0].source, TimelineSource::RedoLog);
871 assert_eq!(report.entries[1].source, TimelineSource::Binlog);
872 }
873
874 #[test]
875 fn test_timeline_source_display() {
876 assert_eq!(TimelineSource::RedoLog.to_string(), "REDO");
877 assert_eq!(TimelineSource::UndoLog.to_string(), "UNDO");
878 assert_eq!(TimelineSource::Binlog.to_string(), "BINLOG");
879 }
880}