1use crate::btree::{OVERFLOW_VLEN, OV_CAP, OV_DATA, OV_NEXT, OV_USED};
10use crate::budget::MemoryBudget;
11use crate::io::{open_file, IoMode};
12use crate::meta::Meta;
13use crate::page::{PageKind, PageRef, PAGE_SIZE};
14use crate::pool::BufferPool;
15use crate::{Error, Result};
16use std::path::Path;
17use std::sync::Arc;
18
19const VERIFY_FRAMES: usize = 16;
20const MAX_TREE_DEPTH: usize = 64;
21
22#[derive(Debug, Clone, Copy)]
23pub(crate) struct VerifiedTree {
24 pub rows: u64,
25 pub pages: u64,
26}
27
28#[derive(Default)]
29struct State {
30 rows: u64,
31 pages: u64,
32 page_count: u32,
33 expected_generation: Option<u64>,
34}
35
36struct Bounds {
37 min: Option<Vec<u8>>,
38 max: Option<Vec<u8>>,
39 first_leaf: u32,
40 last_leaf: u32,
41 last_next: u32,
42}
43
44fn bad(page_no: u32, why: &'static str) -> Error {
45 Error::Corrupt { page_no, why }
46}
47
48fn open_pool(path: &Path, mode: IoMode) -> Result<BufferPool> {
49 let (file, _) = open_file(path, mode)?;
50 let len = file.len()?;
51 if len % PAGE_SIZE as u64 != 0 {
52 return Err(bad(0, "replacement file is not page aligned"));
53 }
54 let pages = len / PAGE_SIZE as u64;
55 if pages < 3 || pages > u32::MAX as u64 {
56 return Err(bad(
57 0,
58 "replacement file page count is outside the format bound",
59 ));
60 }
61 let budget = Arc::new(MemoryBudget::new(VERIFY_FRAMES * PAGE_SIZE));
62 BufferPool::new(file.into(), budget, VERIFY_FRAMES)
63}
64
65pub fn verify_published_tree(path: &Path, mode: IoMode, root: u32, tree_id: u16)
76 -> Result<(u64, u64)>
77{
78 let pool = open_pool(path, mode)?;
79 let mut rows = 0u64;
80 let mut pages = 0u64;
81 let depth_limit = MAX_TREE_DEPTH as u32;
82 published_walk(&pool, root, tree_id, None, None, 0, depth_limit, &mut rows, &mut pages)?;
83 Ok((rows, pages))
84}
85
86#[allow(clippy::too_many_arguments)]
97fn published_walk(
98 pool: &BufferPool,
99 page_no: u32,
100 tree_id: u16,
101 lower: Option<&[u8]>,
102 upper: Option<&[u8]>,
103 depth: u32,
104 depth_limit: u32,
105 rows: &mut u64,
106 pages: &mut u64,
107) -> Result<()> {
108 if depth > depth_limit {
109 return Err(bad(page_no, "published tree exceeds the format depth bound"));
110 }
111 let read = pool.get(page_no)?;
112 let page = crate::page::PageRef::open(&read, page_no)?;
113 if page.tree_id() != tree_id {
114 return Err(bad(page_no, "published page belongs to another tree"));
115 }
116 *pages += 1;
117 if page.kind() == PageKind::Leaf {
118 let mut previous: Option<Vec<u8>> = None;
119 for index in 0..page.nentries() {
120 let record = page.slot(index);
121 let key = match decode_record(record, page_no, PageKind::Leaf)? {
122 DecodedRecord::Leaf { key, value, overflow } => {
123 if overflow {
124 let mut state = State { page_count: pool.page_count(), ..State::default() };
125 verify_overflow(pool, value, &mut state)?;
126 }
127 key
128 },
129 _ => return Err(bad(page_no, "leaf holds an interior record")),
130 };
131 if let Some(previous) = &previous {
132 if key <= previous.as_slice() {
133 return Err(bad(page_no, "published leaf keys are not strictly ordered"));
134 }
135 }
136 if lower.is_some_and(|bound| key < bound) {
137 return Err(bad(page_no, "published leaf key is below its separator"));
138 }
139 if upper.is_some_and(|bound| key >= bound) {
140 return Err(bad(page_no, "published leaf key is at or above its separator"));
141 }
142 previous = Some(key.to_vec());
143 *rows += 1;
144 }
145 return Ok(());
146 }
147 if page.kind() != PageKind::Interior {
148 return Err(bad(page_no, "published tree reaches a non-tree page"));
149 }
150 let mut separators: Vec<(Vec<u8>, u32)> = Vec::with_capacity(page.nentries());
151 for index in 0..page.nentries() {
152 let record = page.slot(index);
153 let DecodedRecord::Interior { key, child } =
154 decode_record(record, page_no, PageKind::Interior)?
155 else {
156 return Err(bad(page_no, "interior page holds a leaf record"));
157 };
158 if separators.last().is_some_and(|(previous, _)| key <= previous.as_slice()) {
159 return Err(bad(page_no, "published separators are not strictly ordered"));
160 }
161 if lower.is_some_and(|bound| key < bound) || upper.is_some_and(|bound| key >= bound) {
162 return Err(bad(page_no, "published separator is outside its parent range"));
163 }
164 separators.push((key.to_vec(), child));
165 }
166 for index in 0..=separators.len() {
167 let child = if index == 0 { page.child0() } else { separators[index - 1].1 };
168 let child_lower = if index == 0 { lower } else { Some(separators[index - 1].0.as_slice()) };
169 let child_upper = if index == separators.len() { upper } else { Some(separators[index].0.as_slice()) };
170 published_walk(pool, child, tree_id, child_lower, child_upper,
171 depth + 1, depth_limit, rows, pages)?;
172 }
173 Ok(())
174}
175
176pub(crate) fn verify_file(
177 path: &Path,
178 mode: IoMode,
179 root: u32,
180 tree_id: u16,
181 expected_rows: u64,
182) -> Result<VerifiedTree> {
183 let pool = open_pool(path, mode)?;
184 verify_tree(&pool, root, tree_id, expected_rows, None, None, 0)
185}
186
187pub(crate) fn verify_range_file(
191 path: &Path,
192 mode: IoMode,
193 root: u32,
194 tree_id: u16,
195 expected_rows: u64,
196 expected_min: &[u8],
197 expected_max: &[u8],
198 expected_next: u32,
199) -> Result<VerifiedTree> {
200 verify_range_file_generation(path, mode, root, tree_id, expected_rows,
201 expected_min, expected_max, expected_next, None)
202}
203
204pub(crate) fn verify_range_file_generation(
205 path: &Path,
206 mode: IoMode,
207 root: u32,
208 tree_id: u16,
209 expected_rows: u64,
210 expected_min: &[u8],
211 expected_max: &[u8],
212 expected_next: u32,
213 expected_generation: Option<u64>,
214) -> Result<VerifiedTree> {
215 let pool = open_pool(path, mode)?;
216 verify_tree_generation(
217 &pool,
218 root,
219 tree_id,
220 expected_rows,
221 Some(expected_min),
222 Some(expected_max),
223 expected_next,
224 expected_generation,
225 )
226}
227
228pub(crate) fn verify_range_pool(
248 pool: &BufferPool,
249 root: u32,
250 tree_id: u16,
251 expected_rows: u64,
252 expected_min: &[u8],
253 expected_max: &[u8],
254 expected_next: u32,
255) -> Result<VerifiedTree> {
256 verify_tree_generation(pool, root, tree_id, expected_rows, Some(expected_min),
257 Some(expected_max), expected_next, None)
258}
259
260pub(crate) fn verify_rebuild(
261 path: &Path,
262 mode: IoMode,
263 expected_meta: &Meta,
264 expected_rows: u64,
265) -> Result<VerifiedTree> {
266 let pool = open_pool(path, mode)?;
267 let meta = Meta::read_latest(&pool)?;
268 if meta.format_version != expected_meta.format_version
269 || meta.roots != expected_meta.roots
270 || meta.next_lsn != expected_meta.next_lsn
271 || meta.generation != expected_meta.generation
272 {
273 return Err(bad(
274 0,
275 "reopened replacement manifest does not match the rebuild",
276 ));
277 }
278 verify_tree(&pool, meta.roots[0], 1, expected_rows, None, None, 0)
279}
280
281pub(crate) fn persist_checkpoint_freelist(
288 dir: &Path,
289 bytes: &[u8],
290 directory_file: &dyn crate::io::FileIo,
291) -> Result<()> {
292 use std::io::Write;
293 let path = dir.join("free");
294 let (mut file, created) = match std::fs::OpenOptions::new().write(true).open(&path) {
295 Ok(file) => (file, false),
296 Err(e) if e.kind() == std::io::ErrorKind::NotFound =>
297 (std::fs::OpenOptions::new().write(true).create_new(true).open(&path)?, true),
298 Err(e) => return Err(e.into()),
299 };
300 file.write_all(bytes)?;
303 file.set_len(bytes.len() as u64)?;
304 crate::write_stats::add(crate::write_stats::Phase::Sidecar, bytes.len() as u64);
305 file.sync_all()?;
306 if created { directory_file.sync_dir()?; }
307 Ok(())
308}
309
310pub(crate) fn publish_freelist(
319 dir: &Path,
320 bytes: Vec<u8>,
321 expected_generation: u64,
322 page_count: u32,
323 directory_file: &dyn crate::io::FileIo,
324 verify_candidate: bool,
325 sync_directory_now: bool,
326) -> Result<()> {
327 let tmp = dir.join("free.tmp");
328 let published = dir.join("free");
329 let result = (|| {
330 std::fs::write(&tmp, &bytes)?;
331 crate::write_stats::add(crate::write_stats::Phase::Sidecar, bytes.len() as u64);
332 std::fs::File::open(&tmp)?.sync_all()?;
333 drop(bytes);
334 if verify_candidate {
335 let reopened = std::fs::read(&tmp)?;
336 if !crate::pool::BufferPool::verify_free(
337 &reopened,
338 expected_generation,
339 page_count,
340 ) {
341 return Err(bad(0, "reopened freelist candidate failed verification"));
342 }
343 }
344 std::fs::rename(&tmp, &published)?;
345 if sync_directory_now {
346 directory_file.sync_dir()?;
347 }
348 Ok(())
349 })();
350 if result.is_err() {
351 let _ = std::fs::remove_file(&tmp);
352 }
353 result
354}
355
356fn verify_tree(
357 pool: &BufferPool,
358 root: u32,
359 tree_id: u16,
360 expected_rows: u64,
361 expected_min: Option<&[u8]>,
362 expected_max: Option<&[u8]>,
363 expected_next: u32,
364) -> Result<VerifiedTree> {
365 verify_tree_generation(pool, root, tree_id, expected_rows, expected_min,
366 expected_max, expected_next, None)
367}
368
369fn verify_tree_generation(
370 pool: &BufferPool,
371 root: u32,
372 tree_id: u16,
373 expected_rows: u64,
374 expected_min: Option<&[u8]>,
375 expected_max: Option<&[u8]>,
376 expected_next: u32,
377 expected_generation: Option<u64>,
378) -> Result<VerifiedTree> {
379 let mut state = State {
380 page_count: pool.page_count(),
381 expected_generation,
382 ..State::default()
383 };
384 let bounds = walk(pool, root, tree_id, None, None, true, 0, &mut state)?;
385 if bounds.last_next != expected_next {
386 return Err(bad(
387 bounds.last_leaf,
388 if expected_next == 0 {
389 "rightmost leaf points past the verified tree"
390 } else {
391 "rightmost leaf does not name the graft continuation"
392 },
393 ));
394 }
395 if state.rows != expected_rows {
396 return Err(bad(
397 root,
398 "replacement row count does not match its manifest",
399 ));
400 }
401 if expected_min.is_some_and(|want| bounds.min.as_deref() != Some(want)) {
402 return Err(bad(root, "replacement minimum does not match its manifest"));
403 }
404 if expected_max.is_some_and(|want| bounds.max.as_deref() != Some(want)) {
405 return Err(bad(root, "replacement maximum does not match its manifest"));
406 }
407 Ok(VerifiedTree {
408 rows: state.rows,
409 pages: state.pages,
410 })
411}
412
413fn visit(state: &mut State, page_no: u32) -> Result<()> {
414 if page_no < 2 || page_no >= state.page_count {
415 return Err(bad(page_no, "replacement graph points outside the file"));
416 }
417 state.pages = state
418 .pages
419 .checked_add(1)
420 .ok_or_else(|| bad(page_no, "replacement page count overflow"))?;
421 if state.pages > state.page_count as u64 {
422 return Err(bad(page_no, "replacement graph cycles or repeats pages"));
423 }
424 Ok(())
425}
426
427fn walk(
428 pool: &BufferPool,
429 page_no: u32,
430 tree_id: u16,
431 lower: Option<&[u8]>,
432 upper: Option<&[u8]>,
433 is_root: bool,
434 depth: usize,
435 state: &mut State,
436) -> Result<Bounds> {
437 if depth > MAX_TREE_DEPTH {
438 return Err(bad(
439 page_no,
440 "replacement tree exceeds the format depth bound",
441 ));
442 }
443 visit(state, page_no)?;
444 let r = pool.get(page_no)?;
445 let page = PageRef::open_resident(&r, page_no)?;
446 if state.expected_generation.is_some_and(|want| page.lsn() != want) {
447 return Err(bad(page_no, "replacement page belongs to another build generation"));
448 }
449 if page.tree_id() != tree_id {
450 return Err(bad(page_no, "replacement page belongs to another tree"));
451 }
452
453 match page.kind() {
454 PageKind::Leaf => verify_leaf(pool, &page, lower, upper, is_root, state),
455 PageKind::Interior => {
456 if page.nentries() == 0 {
457 return Err(bad(
458 page_no,
459 "replacement interior page has fewer than two children",
460 ));
461 }
462 let child0 = page.child0();
463 let mut entries: Vec<(Vec<u8>, u32)> = Vec::with_capacity(page.nentries());
464 for i in 0..page.nentries() {
465 let record = decode_record(page.slot(i), page_no, PageKind::Interior)?;
466 let DecodedRecord::Interior { key, child } = record else { unreachable!() };
467 if i > 0 && entries[i - 1].0.as_slice() >= key {
468 return Err(bad(
469 page_no,
470 "replacement separators are not strictly ordered",
471 ));
472 }
473 if lower.is_some_and(|bound| key < bound) || upper.is_some_and(|bound| key >= bound)
474 {
475 return Err(bad(
476 page_no,
477 "replacement separator is outside its parent range",
478 ));
479 }
480 entries.push((key.to_vec(), child));
481 }
482 drop(r);
483
484 let mut combined: Option<Bounds> = None;
485 for index in 0..=entries.len() {
486 let child = if index == 0 {
487 child0
488 } else {
489 entries[index - 1].1
490 };
491 let child_lower = if index == 0 {
492 lower
493 } else {
494 Some(entries[index - 1].0.as_slice())
495 };
496 let child_upper = if index == entries.len() {
497 upper
498 } else {
499 Some(entries[index].0.as_slice())
500 };
501 let got = walk(
502 pool,
503 child,
504 tree_id,
505 child_lower,
506 child_upper,
507 false,
508 depth + 1,
509 state,
510 )?;
511 if index > 0 && got.min.as_deref() != child_lower {
512 return Err(bad(
513 child,
514 "replacement child minimum does not match its separator",
515 ));
516 }
517 if let Some(acc) = &combined {
518 if acc.last_next != got.first_leaf {
519 return Err(bad(
520 acc.last_leaf,
521 "replacement leaf linkage disagrees with the root graph",
522 ));
523 }
524 }
525 combined = Some(match combined {
526 None => got,
527 Some(acc) => Bounds {
528 min: acc.min,
529 max: got.max,
530 first_leaf: acc.first_leaf,
531 last_leaf: got.last_leaf,
532 last_next: got.last_next,
533 },
534 });
535 }
536 combined.ok_or_else(|| bad(page_no, "replacement interior page has no children"))
537 }
538 _ => Err(bad(
539 page_no,
540 "replacement root graph reaches a non-tree page",
541 )),
542 }
543}
544
545fn verify_leaf(
546 pool: &BufferPool,
547 page: &PageRef<'_>,
548 lower: Option<&[u8]>,
549 upper: Option<&[u8]>,
550 is_root: bool,
551 state: &mut State,
552) -> Result<Bounds> {
553 let page_no = page.page_no();
554 if page.nentries() == 0 && !is_root {
555 return Err(bad(page_no, "replacement contains an empty non-root leaf"));
556 }
557 let mut min = None;
558 let mut max: Option<Vec<u8>> = None;
559 for i in 0..page.nentries() {
560 let record = decode_record(page.slot(i), page_no, PageKind::Leaf)?;
561 let DecodedRecord::Leaf { key, value, overflow } = record else { unreachable!() };
562 if max.as_deref().is_some_and(|previous| previous >= key) {
563 return Err(bad(
564 page_no,
565 "replacement leaf keys are not strictly ordered",
566 ));
567 }
568 if lower.is_some_and(|bound| key < bound) || upper.is_some_and(|bound| key >= bound) {
569 return Err(bad(
570 page_no,
571 "replacement leaf key is outside its parent range",
572 ));
573 }
574 if overflow {
575 verify_overflow(pool, value, state)?;
576 }
577 if min.is_none() {
578 min = Some(key.to_vec());
579 }
580 max = Some(key.to_vec());
581 state.rows = state
582 .rows
583 .checked_add(1)
584 .ok_or_else(|| bad(page_no, "replacement row count overflow"))?;
585 }
586 Ok(Bounds {
587 min,
588 max,
589 first_leaf: page_no,
590 last_leaf: page_no,
591 last_next: page.next_leaf(),
592 })
593}
594
595#[derive(Debug, Clone, Copy)]
601pub enum DecodedRecord<'a> {
602 Leaf { key: &'a [u8], value: &'a [u8], overflow: bool },
603 Interior { key: &'a [u8], child: u32 },
604}
605
606pub fn decode_record(
607 rec: &[u8],
608 page_no: u32,
609 kind: PageKind,
610) -> Result<DecodedRecord<'_>> {
611 if kind==PageKind::Leaf && rec.first()==Some(&0xff)
617 && rec.get(1).is_some_and(|b|(0x81..=0x88).contains(b)) {
618 let end=2+(rec[1]-0x80)as usize;
619 let key=rec.get(1..end).ok_or_else(||bad(page_no,"compact integer key crosses its slot"))?;
620 return Ok(DecodedRecord::Leaf{key,value:&rec[end..],overflow:false});
621 }
622 if kind==PageKind::Leaf && rec.get(1).is_some_and(|b|b&0xf0==0x40) {
623 let end=2+(u16::from_le_bytes([rec[0],rec[1]])&0x0fff) as usize;
624 let key=rec.get(2..end).ok_or_else(||bad(page_no,"compact key crosses its slot"))?;
625 return Ok(DecodedRecord::Leaf{key,value:&rec[end..],overflow:false});
626 }
627 let klen_bytes = rec
628 .get(..2)
629 .ok_or_else(|| bad(page_no, "record has no key length"))?;
630 let klen = u16::from_le_bytes(klen_bytes.try_into().unwrap()) as usize;
631 let key_end = 2usize
632 .checked_add(klen)
633 .ok_or_else(|| bad(page_no, "record key boundary overflow"))?;
634 let key = rec
635 .get(2..key_end)
636 .ok_or_else(|| bad(page_no, "record key crosses its slot"))?;
637
638 if kind == PageKind::Interior {
639 let end = key_end
640 .checked_add(4)
641 .ok_or_else(|| bad(page_no, "interior child boundary overflow"))?;
642 if end != rec.len() {
643 return Err(bad(page_no, "interior record length does not match its slot"));
644 }
645 let child = u32::from_le_bytes(rec[key_end..end].try_into().unwrap());
646 return Ok(DecodedRecord::Interior { key, child });
647 }
648 if kind != PageKind::Leaf {
649 return Err(bad(page_no, "non-tree page contains a tree record"));
650 }
651
652 let vlen_end = key_end
653 .checked_add(2)
654 .ok_or_else(|| bad(page_no, "leaf value boundary overflow"))?;
655 let vlen_bytes = rec
656 .get(key_end..vlen_end)
657 .ok_or_else(|| bad(page_no, "leaf record has no value length"))?;
658 let vlen = u16::from_le_bytes(vlen_bytes.try_into().unwrap());
659 if vlen == OVERFLOW_VLEN {
660 let end = vlen_end
661 .checked_add(12)
662 .ok_or_else(|| bad(page_no, "overflow marker boundary overflow"))?;
663 let marker = rec
664 .get(vlen_end..end)
665 .ok_or_else(|| bad(page_no, "overflow marker crosses its slot"))?;
666 if end != rec.len() {
667 return Err(bad(page_no, "overflow marker has trailing bytes"));
668 }
669 return Ok(DecodedRecord::Leaf { key, value: marker, overflow: true });
670 }
671 let end = vlen_end
672 .checked_add(vlen as usize)
673 .ok_or_else(|| bad(page_no, "leaf value boundary overflow"))?;
674 if end != rec.len() {
675 return Err(bad(page_no, "leaf value length does not match its slot"));
676 }
677 Ok(DecodedRecord::Leaf { key, value: &rec[vlen_end..end], overflow: false })
678}
679
680fn verify_overflow(pool: &BufferPool, marker: &[u8], state: &mut State) -> Result<()> {
681 let total = u32::from_le_bytes(marker[0..4].try_into().unwrap()) as usize;
682 let mut page_no = u32::from_le_bytes(marker[4..8].try_into().unwrap());
683 let want_crc = u32::from_le_bytes(marker[8..12].try_into().unwrap());
684 let expected_pages = total.div_ceil(OV_CAP).max(1);
685 let mut seen = 0usize;
686 let mut bytes = 0usize;
687 let mut crc = 0u32;
688 while page_no != 0 {
689 seen = seen
690 .checked_add(1)
691 .ok_or_else(|| bad(page_no, "overflow page count overflow"))?;
692 if seen > expected_pages {
693 return Err(bad(
694 page_no,
695 "replacement overflow chain cycles or is too long",
696 ));
697 }
698 visit(state, page_no)?;
699 let r = pool.get(page_no)?;
700 let page = PageRef::open_resident(&r, page_no)?;
701 if state.expected_generation.is_some_and(|want| page.lsn() != want) {
702 return Err(bad(page_no, "replacement overflow page belongs to another build generation"));
703 }
704 if page.kind() != PageKind::Overflow || page.tree_id() != 0 {
705 return Err(bad(
706 page_no,
707 "replacement overflow chain reaches the wrong page kind",
708 ));
709 }
710 let used = u16::from_le_bytes(r[OV_USED..OV_USED + 2].try_into().unwrap()) as usize;
711 let next = u32::from_le_bytes(r[OV_NEXT..OV_NEXT + 4].try_into().unwrap());
712 if used > OV_CAP || bytes.checked_add(used).is_none_or(|n| n > total) {
713 return Err(bad(page_no, "replacement overflow length is out of bounds"));
714 }
715 let chunk = &r[OV_DATA..OV_DATA + used];
716 crc = if bytes == 0 {
717 crc32c::crc32c(chunk)
718 } else {
719 crc32c::crc32c_append(crc, chunk)
720 };
721 bytes += used;
722 page_no = next;
723 }
724 if seen != expected_pages || bytes != total || crc != want_crc {
725 return Err(bad(
726 0,
727 "replacement overflow value fails its manifest checksum",
728 ));
729 }
730 Ok(())
731}