use super::{
BinaryCheck, DiscardedProgress, Error, HashSet, NodeState, PackedRecordSet, ProgressObserver,
RecordIdentifier, Result, SegmentProvider, StrideCounter, check_node_shallow, display_relative,
};
pub(crate) struct CorruptLocation {
pub(crate) path: String,
pub(crate) reason: String,
}
pub fn verify_node_tree(provider: &dyn SegmentProvider, root: RecordIdentifier) -> Result<()> {
NodeTreeVerifier::new(provider).verify(root)
}
pub struct NodeTreeVerifier<'provider> {
pub(crate) provider: &'provider dyn SegmentProvider,
pub(crate) verified_subtrees: PackedRecordSet,
pub(crate) verified_nodes: u64,
}
impl<'provider> NodeTreeVerifier<'provider> {
#[must_use]
pub fn new(provider: &'provider dyn SegmentProvider) -> Self {
Self {
provider,
verified_subtrees: PackedRecordSet::new(),
verified_nodes: 0,
}
}
#[must_use]
pub fn verified_nodes(&self) -> u64 {
self.verified_nodes
}
pub fn verify(&mut self, root: RecordIdentifier) -> Result<()> {
self.verify_with_progress(root, &mut DiscardedProgress)
}
pub fn verify_with_progress(
&mut self,
root: RecordIdentifier,
observer: &mut dyn ProgressObserver,
) -> Result<()> {
let mut progress = VerifiedNodeCount::resuming(observer, self.verified_nodes);
let verified = verify_subtree_with_cache(
self.provider,
root,
SubtreeChecks {
binaries: BinaryCheck::EveryBlock,
stable_identifiers: true,
},
&mut self.verified_subtrees,
&mut progress,
);
progress.finish();
self.verified_nodes = progress.completed();
if verified.is_ok() {
assert_eq!(
self.verified_subtrees.len() as u64,
self.verified_nodes,
"the reported node count diverged from the number of certified records"
);
}
verified.map_err(|corrupt| node_tree_error(&corrupt))
}
}
pub(crate) fn node_tree_error(corrupt: &CorruptLocation) -> Error {
Error::InvalidFormat {
details: format!(
"node tree verification failed at {}: {}",
display_relative(&corrupt.path),
corrupt.reason
),
}
}
#[derive(Clone, Copy)]
pub(crate) struct SubtreeChecks {
pub(crate) binaries: BinaryCheck,
pub(crate) stable_identifiers: bool,
}
pub(crate) fn verify_subtree(
provider: &dyn SegmentProvider,
root: RecordIdentifier,
checks: SubtreeChecks,
verified: &mut PackedRecordSet,
progress: &mut VerifiedNodeCount<'_>,
) -> std::result::Result<(), CorruptLocation> {
verify_subtree_with_cache(provider, root, checks, verified, progress)
}
pub(crate) struct VerifiedNodeCount<'observer> {
pub(crate) observer: &'observer mut dyn ProgressObserver,
pub(crate) counter: StrideCounter,
}
impl<'observer> VerifiedNodeCount<'observer> {
pub(in crate::tooling) fn new(observer: &'observer mut dyn ProgressObserver) -> Self {
Self::resuming(observer, 0)
}
pub(in crate::tooling) fn resuming(
observer: &'observer mut dyn ProgressObserver,
already: u64,
) -> Self {
Self {
observer,
counter: StrideCounter::resuming(VERIFIED_NODE_REPORT_STRIDE, already),
}
}
pub(in crate::tooling) fn completed(&self) -> u64 {
self.counter.completed()
}
pub(in crate::tooling) fn advance(&mut self) {
self.counter.advance(self.observer);
}
pub(in crate::tooling) fn finish(&mut self) {
self.counter.finish(self.observer);
}
}
pub(crate) const VERIFIED_NODE_REPORT_STRIDE: u64 = 512;
pub(crate) struct VerificationFrame {
pub(crate) record: RecordIdentifier,
pub(crate) pending_children: Vec<(String, RecordIdentifier)>,
pub(crate) parent_path_length: usize,
}
pub(crate) fn open(
provider: &dyn SegmentProvider,
record: RecordIdentifier,
checks: SubtreeChecks,
verified: &PackedRecordSet,
ancestors: &mut HashSet<RecordIdentifier>,
path: &str,
) -> std::result::Result<Option<Vec<(String, RecordIdentifier)>>, CorruptLocation> {
let corrupt_here = |reason: String| CorruptLocation {
path: path.to_owned(),
reason,
};
if ancestors.contains(&record) {
return Err(corrupt_here(format!(
"node record {record} is contained in its own subtree"
)));
}
if verified.contains(record) {
return Ok(None);
}
ancestors.insert(record);
if let Err(reason) = check_node_shallow(provider, record, checks.binaries) {
return Err(CorruptLocation {
path: path.to_owned(),
reason,
});
}
let node = NodeState::new(provider, record);
if checks.stable_identifiers
&& let Err(error) = node.stable_identifier_bytes()
{
return Err(CorruptLocation {
path: path.to_owned(),
reason: error.to_string(),
});
}
let mut children: Vec<(String, RecordIdentifier)> = node
.child_node_entries()
.map_err(|error| corrupt_here(error.to_string()))?
.into_iter()
.map(|(name, child)| (name, child.record_identifier()))
.collect();
children.reverse();
Ok(Some(children))
}
pub(crate) fn verify_subtree_with_cache(
provider: &dyn SegmentProvider,
root: RecordIdentifier,
checks: SubtreeChecks,
verified: &mut PackedRecordSet,
progress: &mut VerifiedNodeCount<'_>,
) -> std::result::Result<(), CorruptLocation> {
let mut ancestors = HashSet::new();
let mut path = String::new();
let Some(children) = open(provider, root, checks, verified, &mut ancestors, &path)? else {
return Ok(());
};
let mut stack = vec![VerificationFrame {
record: root,
pending_children: children,
parent_path_length: 0,
}];
loop {
let next = stack
.last_mut()
.expect("the loop returns before the stack empties")
.pending_children
.pop();
if let Some((name, child)) = next {
let parent_path_length = path.len();
path.push('/');
path.push_str(&name);
match open(provider, child, checks, verified, &mut ancestors, &path)? {
Some(children) => stack.push(VerificationFrame {
record: child,
pending_children: children,
parent_path_length,
}),
None => path.truncate(parent_path_length),
}
continue;
}
let finished = stack.pop().expect("a frame was just inspected");
ancestors.remove(&finished.record);
verified.insert(finished.record);
progress.advance();
if stack.is_empty() {
return Ok(());
}
path.truncate(finished.parent_path_length);
}
}
pub(crate) fn materialize_binary(
provider: &dyn SegmentProvider,
value: &crate::content::property::PropertyValue,
) -> Result<()> {
use crate::content::property::PropertyValue;
use crate::content::value::BinaryValue;
if let PropertyValue::Binary(BinaryValue::Inline {
record_identifier, ..
}) = value
{
crate::content::value::verify_binary_content(provider, *record_identifier)?;
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::{NodeTreeVerifier, verify_node_tree};
use crate::content::node::NodeState;
use crate::content::provider::SegmentProvider;
use crate::content::provider::tests::MemorySegmentProvider;
use crate::error::Error;
use crate::progress::{ProgressObserver, Step};
use crate::segment::record::RecordIdentifier;
use crate::tooling::check::test_support::{CountingProvider, HidingProvider, TestDirectory};
use crate::writer::record_writer::{ChildNodesToWrite, PropertyToWrite, PropertyValuesToWrite};
use crate::writer::segment_builder::SegmentBufferBuilder;
use crate::writer::store_writer::WritableRepository;
#[test]
fn node_tree_verifier_reports_the_corrupt_descendant_path() {
let directory = TestDirectory::new("verify-corrupt-path");
let store = WritableRepository::open(&directory.path).expect("open");
let generation = store.writing_generation().expect("generation");
let mut child_writer = store.record_writer(generation);
let child = child_writer
.write_node(None, &[], &ChildNodesToWrite::Zero, &[])
.expect("child");
child_writer.finish().expect("finish child");
let mut root_writer = store.record_writer(generation);
let root = root_writer
.write_node(
None,
&[],
&ChildNodesToWrite::One {
name: "broken".to_owned(),
node: child,
},
&[],
)
.expect("root");
root_writer.finish().expect("finish root");
verify_node_tree(&store, root).expect("the complete tree verifies");
let provider = HidingProvider {
store: &store,
exact: Some(child.segment),
bulk: false,
};
let error = verify_node_tree(&provider, root).expect_err("hidden child must fail");
let Error::InvalidFormat { details } = error else {
panic!("verification must return a structured format error");
};
assert!(
details.contains("at /broken:"),
"the error identifies the corrupt relative path: {details}"
);
assert!(
details.contains(&child.segment.to_string()),
"the underlying failure remains useful: {details}"
);
let provider = HidingProvider {
store: &store,
exact: Some(root.segment),
bulk: false,
};
let error = verify_node_tree(&provider, root).expect_err("hidden root must fail");
let Error::InvalidFormat { details } = error else {
panic!("verification must return a structured format error");
};
assert!(
details.contains("at /:"),
"root corruption uses the documented root path: {details}"
);
store.close().expect("close");
}
#[test]
fn reusable_node_tree_verifier_reuses_fully_verified_shared_descendants() {
let directory = TestDirectory::new("verify-shared-cache");
let store = WritableRepository::open(&directory.path).expect("open");
let generation = store.writing_generation().expect("generation");
let mut child_writer = store.record_writer(generation);
let shared = child_writer
.write_node(None, &[], &ChildNodesToWrite::Zero, &[])
.expect("shared child");
child_writer.finish().expect("finish shared child");
let mut first_writer = store.record_writer(generation);
let first_root = first_writer
.write_node(
None,
&[],
&ChildNodesToWrite::One {
name: "shared".to_owned(),
node: shared,
},
&[],
)
.expect("first root");
first_writer.finish().expect("finish first root");
let mut second_writer = store.record_writer(generation);
let second_root = second_writer
.write_node(
None,
&[],
&ChildNodesToWrite::One {
name: "shared".to_owned(),
node: shared,
},
&[],
)
.expect("second root");
second_writer.finish().expect("finish second root");
let provider = CountingProvider::new(&store);
let mut verifier = NodeTreeVerifier::new(&provider);
verifier.verify(first_root).expect("first tree verifies");
let shared_reads = provider.reads_of(shared.segment);
assert!(shared_reads > 0, "the first root traverses the shared node");
assert!(
verifier.verified_subtrees.contains(shared),
"only a completed shared subtree receives a certificate"
);
verifier.verify(second_root).expect("second tree verifies");
assert_eq!(
provider.reads_of(shared.segment),
shared_reads,
"the second root reuses the provider-bound subtree certificate"
);
store.close().expect("close");
}
#[derive(Default)]
struct HighestReportedCount {
highest: u64,
}
impl ProgressObserver for HighestReportedCount {
fn step_began(&mut self, _step: &Step<'_>) {}
fn step_advanced(&mut self, completed: u64) {
assert!(
completed >= self.highest,
"a running total must never go backwards: {completed} after {}",
self.highest
);
self.highest = completed;
}
fn step_ended(&mut self) {}
}
fn distinct_nodes_below(
provider: &dyn SegmentProvider,
roots: &[RecordIdentifier],
) -> std::collections::HashSet<RecordIdentifier> {
let mut seen = std::collections::HashSet::new();
let mut pending: Vec<RecordIdentifier> = roots.to_vec();
while let Some(record) = pending.pop() {
if !seen.insert(record) {
continue;
}
for (_, child) in NodeState::new(provider, record)
.child_node_entries()
.expect("enumerate the child nodes")
{
pending.push(child.record_identifier());
}
}
seen
}
#[test]
fn a_super_root_with_checkpoint_snapshots_counts_each_distinct_node_once() {
let directory = TestDirectory::new("verify-exact-count");
let store = WritableRepository::open(&directory.path).expect("open");
let generation = store.writing_generation().expect("generation");
let mut writer = store.record_writer(generation);
let mut level = Vec::new();
for leaf in 0..64 {
let value = writer
.write_string(&format!("leaf-{leaf}"))
.expect("leaf value");
let node = writer
.write_node(
None,
&[],
&ChildNodesToWrite::Zero,
&[PropertyToWrite {
name: "data".to_owned(),
property_type: crate::content::property::PropertyType::String,
values: PropertyValuesToWrite::Single(value),
}],
)
.expect("leaf node");
level.push((format!("leaf{leaf}"), node));
}
let mut content_root = writer
.write_node(None, &[], &ChildNodesToWrite::Many(level), &[])
.expect("content root");
for depth in 0..8 {
content_root = writer
.write_node(
None,
&[],
&ChildNodesToWrite::One {
name: format!("level{depth}"),
node: content_root,
},
&[],
)
.expect("branch node");
}
let mut snapshots = Vec::new();
for snapshot in 0..2 {
snapshots.push((
format!("checkpoint{snapshot}"),
writer
.write_node(
None,
&[],
&ChildNodesToWrite::One {
name: "root".to_owned(),
node: content_root,
},
&[],
)
.expect("checkpoint snapshot"),
));
}
let checkpoints = writer
.write_node(None, &[], &ChildNodesToWrite::Many(snapshots), &[])
.expect("checkpoint container");
let super_root = writer
.write_node(
None,
&[],
&ChildNodesToWrite::Many(vec![
("root".to_owned(), content_root),
("checkpoints".to_owned(), checkpoints),
]),
&[],
)
.expect("super root");
writer.finish().expect("finish");
let expected = distinct_nodes_below(&store, &[super_root]).len();
assert!(
expected > 64,
"the fixture must be large enough to matter, got {expected} nodes"
);
let mut reported = HighestReportedCount::default();
NodeTreeVerifier::new(&store)
.verify_with_progress(super_root, &mut reported)
.expect("the super root verifies");
assert_eq!(
reported.highest, expected as u64,
"every distinct node is reported once, however many roots reach it"
);
store.close().expect("close");
}
#[test]
fn reusable_node_tree_verifier_never_caches_a_failed_subtree() {
let directory = TestDirectory::new("verify-failed-not-cached");
let store = WritableRepository::open(&directory.path).expect("open");
let generation = store.writing_generation().expect("generation");
let mut child_writer = store.record_writer(generation);
let child = child_writer
.write_node(None, &[], &ChildNodesToWrite::Zero, &[])
.expect("child");
child_writer.finish().expect("finish child");
let mut root_writer = store.record_writer(generation);
let root = root_writer
.write_node(
None,
&[],
&ChildNodesToWrite::One {
name: "broken".to_owned(),
node: child,
},
&[],
)
.expect("root");
root_writer.finish().expect("finish root");
let provider = CountingProvider::hiding(&store, child.segment);
let mut verifier = NodeTreeVerifier::new(&provider);
let first = verifier.verify(root).expect_err("hidden child fails");
let first_reads = provider.reads_of(child.segment);
assert!(first_reads > 0);
assert!(
verifier.verified_subtrees.len() == 0,
"neither the corrupt child nor its incomplete ancestor is cached"
);
let second = verifier
.verify(root)
.expect_err("the same hidden child must be re-read and fail again");
assert!(provider.reads_of(child.segment) > first_reads);
assert_eq!(first.to_string(), second.to_string());
assert_eq!(verifier.verified_subtrees.len(), 0);
store.close().expect("close");
}
#[test]
fn reusable_node_tree_verifier_never_caches_a_cyclic_subtree() {
let directory = TestDirectory::new("verify-cycle-not-cached");
let store = WritableRepository::open(&directory.path).expect("open");
let generation = store.writing_generation().expect("generation");
let mut writer = store.record_writer(generation);
let original_child = writer
.write_node(None, &[], &ChildNodesToWrite::Zero, &[])
.expect("original child");
let root = writer
.write_node(
None,
&[],
&ChildNodesToWrite::One {
name: "loop".to_owned(),
node: original_child,
},
&[],
)
.expect("root");
writer.finish().expect("finish segment");
let view = store.segment(root.segment).expect("root segment");
let root_position = view
.record_position(root.record_number)
.expect("root position");
let mut cyclic_bytes = view.bytes.to_vec();
let child_slot: &mut [u8; 6] = (&mut cyclic_bytes[root_position + 12..root_position + 18])
.try_into()
.expect("one child identifier slot");
SegmentBufferBuilder::write_record_identifier_bytes(0, root.record_number, child_slot);
let mut memory = MemorySegmentProvider::default();
memory.insert(root.segment, cyclic_bytes);
let provider = CountingProvider::new(&memory);
let mut verifier = NodeTreeVerifier::new(&provider);
let first = verifier.verify(root).expect_err("self-cycle fails");
let first_reads = provider.reads_of(root.segment);
let Error::InvalidFormat { details } = &first else {
panic!("cycle verification returns a format error");
};
assert!(details.contains("at /loop:"));
assert!(details.contains("contained in its own subtree"));
assert_eq!(verifier.verified_subtrees.len(), 0);
let second = verifier.verify(root).expect_err("cycle is never cached");
assert!(provider.reads_of(root.segment) > first_reads);
assert_eq!(first.to_string(), second.to_string());
assert_eq!(verifier.verified_subtrees.len(), 0);
store.close().expect("close");
}
#[test]
fn node_tree_verifier_materializes_long_inline_binary_blocks() {
let directory = TestDirectory::new("verify-inline-binary");
let store = WritableRepository::open(&directory.path).expect("open");
let generation = store.writing_generation().expect("generation");
let content: Vec<u8> = (0..300 * 1024).map(|index| (index % 251) as u8).collect();
let mut writer = store.record_writer(generation);
let binary = writer.write_binary_content(&content).expect("binary");
let payload = writer
.write_node(
None,
&[],
&ChildNodesToWrite::Zero,
&[PropertyToWrite {
name: "data".to_owned(),
property_type: crate::content::property::PropertyType::Binary,
values: PropertyValuesToWrite::Single(binary),
}],
)
.expect("payload");
let root = writer
.write_node(
None,
&[],
&ChildNodesToWrite::One {
name: "payload".to_owned(),
node: payload,
},
&[],
)
.expect("root");
writer.finish().expect("finish");
verify_node_tree(&store, root).expect("complete binary verifies");
let provider = HidingProvider {
store: &store,
exact: None,
bulk: true,
};
let error = verify_node_tree(&provider, root).expect_err("missing block must fail");
let Error::InvalidFormat { details } = error else {
panic!("verification must return a structured format error");
};
assert!(
details.contains("at /payload:"),
"binary corruption is attributed to its containing node: {details}"
);
assert!(
details.contains("not found in any archive"),
"the missing block reason is retained: {details}"
);
store.close().expect("close");
}
#[test]
fn node_tree_verifier_resolves_preserved_stable_identifiers() {
let directory = TestDirectory::new("verify-stable-identifier");
let store = WritableRepository::open(&directory.path).expect("open");
let generation = store.writing_generation().expect("generation");
let mut child_writer = store.record_writer(generation);
let child = child_writer
.write_node_with_stable_identifier(
None,
&[],
&ChildNodesToWrite::Zero,
&[],
Some([0x5a; 20]),
)
.expect("child");
child_writer.finish().expect("finish child");
let mut root_writer = store.record_writer(generation);
let root = root_writer
.write_node(
None,
&[],
&ChildNodesToWrite::One {
name: "stable".to_owned(),
node: child,
},
&[],
)
.expect("root");
root_writer.finish().expect("finish root");
verify_node_tree(&store, root).expect("valid stable identifier verifies");
let child_view = store.segment(child.segment).expect("child segment");
let mut child_bytes = child_view.bytes.to_vec();
let child_position = child_view
.record_position(child.record_number)
.expect("child position");
child_bytes[child_position..child_position + 2].copy_from_slice(&0u16.to_be_bytes());
child_bytes[child_position + 2..child_position + 6]
.copy_from_slice(&u32::MAX.to_be_bytes());
let root_view = store.segment(root.segment).expect("root segment");
let mut provider = MemorySegmentProvider::default();
provider.insert(child.segment, child_bytes);
provider.insert(root.segment, root_view.bytes.to_vec());
let error = verify_node_tree(&provider, root).expect_err("invalid stable id must fail");
let Error::InvalidFormat { details } = error else {
panic!("verification must return a structured format error");
};
assert!(
details.contains("at /stable:"),
"stable-id corruption is attributed to its node: {details}"
);
assert!(
details.contains("record 4294967295 does not exist"),
"the stable-id failure reason is retained: {details}"
);
store.close().expect("close");
}
}