#[cfg(feature = "alloc")]
use alloc::{vec, vec::Vec};
use core::fmt;
use oxgraph_layout_util::SnapshotWidth;
use zerocopy::{
FromBytes, Immutable, IntoBytes, KnownLayout,
byteorder::{LE, U32, U64},
};
use crate::container_error::{PlanError, SectionBindError, SectionViewError, SnapshotError};
pub const FORMAT_MAGIC: [u8; 8] = *b"OXGTOPO\0";
pub const FORMAT_MAJOR: u32 = 1;
pub const FORMAT_MINOR: u32 = 0;
pub const MAX_SUPPORTED_MINOR: u32 = 0;
pub const HEADER_SIZE: usize = 32;
pub const SECTION_ENTRY_SIZE: usize = 32;
pub const MAX_ALIGNMENT_LOG2: u8 = 12;
pub const MAX_SECTION_COUNT: u32 = 1024;
const HEADER_SIZE_U32: u32 = 32;
#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
pub struct SectionKind(u32);
impl SectionKind {
#[must_use]
pub const fn new(value: u32) -> Self {
Self(value)
}
#[must_use]
pub const fn get(self) -> u32 {
self.0
}
}
impl From<u32> for SectionKind {
fn from(value: u32) -> Self {
Self(value)
}
}
impl fmt::Display for SectionKind {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(formatter, "{:#06x}", self.0)
}
}
pub mod kinds {
use core::ops::Range;
pub const CSR_BAND: Range<u32> = 0x0001..0x0020;
pub const BCSR_BAND: Range<u32> = 0x0020..0x0100;
pub const PROPERTY_BAND: Range<u32> = 0x0100..0x0200;
pub const POSTGRES_BAND: Range<u32> = 0x0200..0x0300;
pub const DATABASE_BAND: Range<u32> = 0x0300..0x0400;
pub const CUSTOM_BASE: u32 = 0x0400;
#[must_use]
pub const fn in_band(kind: u32, band: Range<u32>) -> bool {
kind >= band.start && kind < band.end
}
}
fn u64_to_usize_validated(value: u64) -> usize {
match usize::try_from(value) {
Ok(converted) => converted,
Err(_error) => unreachable!("validated u64 must fit usize on this target"),
}
}
#[derive(Clone, Copy, Debug, FromBytes, Immutable, IntoBytes, KnownLayout)]
#[repr(C)]
struct RawHeader {
magic: [u8; 8],
format_major: U32<LE>,
format_minor: U32<LE>,
header_size: U32<LE>,
section_count: U32<LE>,
reserved: [u8; 8],
}
fn parse_header(bytes: &[u8]) -> Result<(&RawHeader, &[u8]), SnapshotError> {
if bytes.len() < HEADER_SIZE {
return Err(SnapshotError::TruncatedHeader {
needed: HEADER_SIZE,
actual: bytes.len(),
});
}
match RawHeader::ref_from_prefix(bytes) {
Ok((header, rest)) => Ok((header, rest)),
Err(_error) => Err(SnapshotError::MalformedHeader),
}
}
fn validate_magic_versions_reserved(header: &RawHeader) -> Result<(), SnapshotError> {
if header.magic != FORMAT_MAGIC {
return Err(SnapshotError::BadMagic {
actual: header.magic,
});
}
let major = header.format_major.get();
if major != FORMAT_MAJOR {
return Err(SnapshotError::FormatMajorMismatch {
actual: major,
supported: FORMAT_MAJOR,
});
}
let minor = header.format_minor.get();
if minor > MAX_SUPPORTED_MINOR {
return Err(SnapshotError::FormatMinorTooNew {
actual: minor,
max_supported: MAX_SUPPORTED_MINOR,
});
}
let header_size = header.header_size.get();
if header_size != HEADER_SIZE_U32 {
return Err(SnapshotError::HeaderSizeMismatch {
actual: header_size,
expected: HEADER_SIZE_U32,
});
}
if header.reserved != [0; 8] {
return Err(SnapshotError::NonZeroHeaderReserved);
}
Ok(())
}
#[derive(Clone, Copy, Debug, FromBytes, Immutable, IntoBytes, KnownLayout)]
#[repr(C)]
struct RawSectionEntry {
offset: U64<LE>,
length: U64<LE>,
kind: U32<LE>,
version: U32<LE>,
reserved_checksum: [u8; 4],
alignment_log2: u8,
flags: u8,
reserved: [u8; 2],
}
#[derive(Clone, Copy, Debug)]
pub struct Section<'view> {
payload: &'view [u8],
kind: u32,
version: u32,
alignment_log2: u8,
}
impl<'view> Section<'view> {
#[must_use]
fn from_entry(bytes: &'view [u8], entry: &RawSectionEntry) -> Self {
let offset = u64_to_usize_validated(entry.offset.get());
let length = u64_to_usize_validated(entry.length.get());
Self {
payload: &bytes[offset..offset + length],
kind: entry.kind.get(),
version: entry.version.get(),
alignment_log2: entry.alignment_log2,
}
}
#[must_use]
pub const fn kind(&self) -> u32 {
self.kind
}
#[must_use]
pub const fn version(&self) -> u32 {
self.version
}
#[must_use]
pub const fn declared_alignment(&self) -> usize {
1usize << self.alignment_log2
}
#[must_use]
pub const fn bytes(&self) -> &'view [u8] {
self.payload
}
pub fn try_as_slice<T>(&self) -> Result<&'view [T], SectionViewError>
where
T: zerocopy::FromBytes + zerocopy::Immutable + zerocopy::KnownLayout,
{
let elem_size = core::mem::size_of::<T>();
let length = self.payload.len();
if elem_size == 0 {
return Err(SectionViewError::ZeroSizedType);
}
if !length.is_multiple_of(elem_size) {
return Err(SectionViewError::LengthNotMultipleOfSize { length, elem_size });
}
let required = core::mem::align_of::<T>();
let ptr_addr = self.payload.as_ptr().addr();
if !ptr_addr.is_multiple_of(required) {
return Err(SectionViewError::AlignmentMismatch { ptr_addr, required });
}
let count = length / elem_size;
match <[T]>::ref_from_bytes_with_elems(self.payload, count) {
Ok(slice) => Ok(slice),
Err(_error) => Err(SectionViewError::AlignmentMismatch { ptr_addr, required }),
}
}
}
#[non_exhaustive]
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum ValidationLevel {
SectionTable,
Layout,
}
fn first_duplicate_kind<T>(items: &[T], kind_of: impl Fn(&T) -> u32) -> Option<u32> {
for (index, item) in items.iter().enumerate() {
let kind = kind_of(item);
if items[..index].iter().any(|prior| kind_of(prior) == kind) {
return Some(kind);
}
}
None
}
fn validate_section_table(
bytes: &[u8],
entries: &[RawSectionEntry],
level: ValidationLevel,
) -> Result<(), SnapshotError> {
let snapshot_len = bytes.len() as u64;
for entry in entries {
let kind = entry.kind.get();
if entry.reserved_checksum != [0; 4] {
return Err(SnapshotError::NonZeroEntryChecksum { kind });
}
if entry.flags != 0 {
return Err(SnapshotError::UnsupportedFlags {
kind,
flags: entry.flags,
});
}
if entry.reserved != [0; 2] {
return Err(SnapshotError::NonZeroEntryReserved { kind });
}
if entry.alignment_log2 > MAX_ALIGNMENT_LOG2 {
return Err(SnapshotError::AlignmentLog2TooLarge {
kind,
alignment_log2: entry.alignment_log2,
});
}
let offset = entry.offset.get();
let length = entry.length.get();
let end = offset
.checked_add(length)
.ok_or(SnapshotError::SectionRangeOverflow { kind })?;
if end > snapshot_len {
return Err(SnapshotError::SectionOutOfBounds {
kind,
offset,
length,
snapshot_len,
});
}
}
if matches!(level, ValidationLevel::SectionTable) {
return Ok(());
}
let header_plus_table = (HEADER_SIZE as u64)
.checked_add((entries.len() as u64).saturating_mul(SECTION_ENTRY_SIZE as u64))
.ok_or(SnapshotError::SectionRangeOverflow { kind: 0 })?;
let mut prev_end = header_plus_table;
for (index, entry) in entries.iter().enumerate() {
let offset = entry.offset.get();
let end = offset.saturating_add(entry.length.get());
if offset < prev_end {
return Err(SnapshotError::UnsortedSectionTable { index });
}
prev_end = end;
}
if let Some(kind) = first_duplicate_kind(entries, |entry| entry.kind.get()) {
return Err(SnapshotError::DuplicateKind { kind });
}
Ok(())
}
#[derive(Clone, Copy, Debug)]
pub struct HeaderOnlySnapshot<'view> {
bytes: &'view [u8],
format_major: u32,
format_minor: u32,
}
impl<'view> HeaderOnlySnapshot<'view> {
pub fn open(bytes: &'view [u8]) -> Result<Self, SnapshotError> {
let (header, _after_header) = parse_header(bytes)?;
validate_magic_versions_reserved(header)?;
Ok(Self {
bytes,
format_major: header.format_major.get(),
format_minor: header.format_minor.get(),
})
}
#[must_use]
pub const fn bytes(&self) -> &'view [u8] {
self.bytes
}
#[must_use]
pub const fn format_major(&self) -> u32 {
self.format_major
}
#[must_use]
pub const fn format_minor(&self) -> u32 {
self.format_minor
}
}
#[derive(Clone, Copy, Debug)]
pub struct Snapshot<'view> {
bytes: &'view [u8],
format_major: u32,
format_minor: u32,
entries: &'view [RawSectionEntry],
}
impl<'view> Snapshot<'view> {
pub fn open(bytes: &'view [u8]) -> Result<Self, SnapshotError> {
Self::open_with(bytes, ValidationLevel::Layout)
}
pub fn open_with(bytes: &'view [u8], level: ValidationLevel) -> Result<Self, SnapshotError> {
let (header, after_header) = parse_header(bytes)?;
validate_magic_versions_reserved(header)?;
let format_major = header.format_major.get();
let format_minor = header.format_minor.get();
let section_count = header.section_count.get();
if section_count > MAX_SECTION_COUNT {
return Err(SnapshotError::SectionCountTooLarge {
count: section_count,
max: MAX_SECTION_COUNT,
});
}
let Ok(section_count_usize) = usize::try_from(section_count) else {
return Err(SnapshotError::UsizeOverflow {
value: u64::from(section_count),
});
};
let Some(table_len) = section_count_usize.checked_mul(SECTION_ENTRY_SIZE) else {
return Err(SnapshotError::SectionCountTooLarge {
count: section_count,
max: MAX_SECTION_COUNT,
});
};
if after_header.len() < table_len {
return Err(SnapshotError::TruncatedSectionTable {
needed: table_len,
actual: after_header.len(),
});
}
let table_bytes = &after_header[..table_len];
let entries =
<[RawSectionEntry]>::ref_from_bytes_with_elems(table_bytes, section_count_usize)
.map_err(|_error| SnapshotError::MalformedSectionTable)?;
validate_section_table(bytes, entries, level)?;
Ok(Self {
bytes,
format_major,
format_minor,
entries,
})
}
#[must_use]
pub const fn format_major(&self) -> u32 {
self.format_major
}
#[must_use]
pub const fn format_minor(&self) -> u32 {
self.format_minor
}
#[must_use]
pub const fn section_count(&self) -> usize {
self.entries.len()
}
#[must_use]
pub fn sections(&self) -> SectionIter<'view> {
SectionIter {
bytes: self.bytes,
entries: self.entries.iter(),
}
}
#[must_use]
pub fn section(&self, kind: u32) -> Option<Section<'view>> {
let bytes = self.bytes;
self.entries.iter().find_map(|entry| {
if entry.kind.get() == kind {
Some(Section::from_entry(bytes, entry))
} else {
None
}
})
}
pub fn typed_section<W>(
&self,
kind: u32,
expected_version: u32,
) -> Result<&'view [W::LittleEndianWord], SectionBindError>
where
W: SnapshotWidth,
{
let section = self
.section(kind)
.ok_or(SectionBindError::Missing { kind })?;
if section.version() != expected_version {
return Err(SectionBindError::VersionMismatch {
kind,
expected: expected_version,
actual: section.version(),
});
}
section
.try_as_slice::<W::LittleEndianWord>()
.map_err(|error| SectionBindError::View { kind, error })
}
}
#[derive(Clone, Debug)]
pub struct SectionIter<'view> {
bytes: &'view [u8],
entries: core::slice::Iter<'view, RawSectionEntry>,
}
impl<'view> Iterator for SectionIter<'view> {
type Item = Section<'view>;
fn next(&mut self) -> Option<Self::Item> {
self.entries
.next()
.map(|entry| Section::from_entry(self.bytes, entry))
}
fn size_hint(&self) -> (usize, Option<usize>) {
self.entries.size_hint()
}
}
impl ExactSizeIterator for SectionIter<'_> {
fn len(&self) -> usize {
self.entries.len()
}
}
#[derive(Clone, Copy, Debug)]
pub struct PendingSection<'a> {
pub kind: u32,
pub version: u32,
pub alignment_log2: u8,
pub payload: &'a [u8],
}
#[derive(Clone, Copy, Debug)]
pub struct SnapshotPlan<'a> {
sections: &'a [PendingSection<'a>],
}
impl<'a> SnapshotPlan<'a> {
pub fn new(sections: &'a [PendingSection<'a>]) -> Result<Self, PlanError> {
if sections.len() > MAX_SECTION_COUNT as usize {
return Err(PlanError::TooManySections {
count: sections.len(),
});
}
for section in sections {
if section.alignment_log2 > MAX_ALIGNMENT_LOG2 {
return Err(PlanError::AlignmentTooLarge {
alignment_log2: section.alignment_log2,
});
}
}
if let Some(kind) = first_duplicate_kind(sections, |section| section.kind) {
return Err(PlanError::DuplicateKind { kind });
}
Ok(Self { sections })
}
#[must_use]
pub const fn section_count(&self) -> usize {
self.sections.len()
}
pub fn encoded_len(&self) -> Result<usize, PlanError> {
let table_len = self
.sections
.len()
.checked_mul(SECTION_ENTRY_SIZE)
.ok_or(PlanError::PayloadOverflow)?;
let mut total = HEADER_SIZE
.checked_add(table_len)
.ok_or(PlanError::PayloadOverflow)?;
for section in self.sections {
total = align_up_checked(total, section.alignment_log2)?;
total = total
.checked_add(section.payload.len())
.ok_or(PlanError::PayloadOverflow)?;
}
u64::try_from(total).map_err(|_error| PlanError::PayloadOverflow)?;
Ok(total)
}
pub fn write_into(&self, out: &mut [u8]) -> Result<usize, PlanError> {
let needed = self.encoded_len()?;
if out.len() < needed {
return Err(PlanError::BufferTooSmall {
needed,
actual: out.len(),
});
}
let prefix = &mut out[..needed];
prefix.fill(0);
let section_count_u32 = match u32::try_from(self.sections.len()) {
Ok(value) => value,
Err(_error) => {
return Err(PlanError::TooManySections {
count: self.sections.len(),
});
}
};
let header = RawHeader {
magic: FORMAT_MAGIC,
format_major: U32::new(FORMAT_MAJOR),
format_minor: U32::new(FORMAT_MINOR),
header_size: U32::new(HEADER_SIZE_U32),
section_count: U32::new(section_count_u32),
reserved: [0; 8],
};
prefix[..HEADER_SIZE].copy_from_slice(header.as_bytes());
let table_start = HEADER_SIZE;
let payload_start = table_start
.checked_add(
self.sections
.len()
.checked_mul(SECTION_ENTRY_SIZE)
.ok_or(PlanError::PayloadOverflow)?,
)
.ok_or(PlanError::PayloadOverflow)?;
let mut cursor = payload_start;
for (index, section) in self.sections.iter().enumerate() {
cursor = align_up_checked(cursor, section.alignment_log2)?;
let payload_end = cursor
.checked_add(section.payload.len())
.ok_or(PlanError::PayloadOverflow)?;
let offset_u64 = u64::try_from(cursor).map_err(|_error| PlanError::PayloadOverflow)?;
let length_u64 = u64::try_from(section.payload.len())
.map_err(|_error| PlanError::PayloadOverflow)?;
let entry = RawSectionEntry {
offset: U64::new(offset_u64),
length: U64::new(length_u64),
kind: U32::new(section.kind),
version: U32::new(section.version),
reserved_checksum: [0; 4],
alignment_log2: section.alignment_log2,
flags: 0,
reserved: [0; 2],
};
let entry_offset = table_start
.checked_add(
index
.checked_mul(SECTION_ENTRY_SIZE)
.ok_or(PlanError::PayloadOverflow)?,
)
.ok_or(PlanError::PayloadOverflow)?;
prefix[entry_offset..entry_offset + SECTION_ENTRY_SIZE]
.copy_from_slice(entry.as_bytes());
prefix[cursor..payload_end].copy_from_slice(section.payload);
cursor = payload_end;
}
Ok(needed)
}
}
fn align_up_checked(value: usize, alignment_log2: u8) -> Result<usize, PlanError> {
let alignment = 1usize << alignment_log2;
let mask = alignment - 1;
let added = value.checked_add(mask).ok_or(PlanError::PayloadOverflow)?;
Ok(added & !mask)
}
#[cfg(feature = "alloc")]
#[derive(Clone, Debug)]
struct OwnedSection {
kind: u32,
version: u32,
alignment_log2: u8,
payload: Vec<u8>,
}
#[cfg(feature = "alloc")]
#[derive(Clone, Debug, Default)]
#[must_use]
pub struct SnapshotBuilder {
sections: Vec<OwnedSection>,
}
#[cfg(feature = "alloc")]
impl SnapshotBuilder {
pub fn new() -> Self {
Self::default()
}
pub fn add_section(
&mut self,
kind: u32,
version: u32,
alignment_log2: u8,
payload: Vec<u8>,
) -> Result<&mut Self, PlanError> {
if alignment_log2 > MAX_ALIGNMENT_LOG2 {
return Err(PlanError::AlignmentTooLarge { alignment_log2 });
}
if self.sections.len() >= MAX_SECTION_COUNT as usize {
return Err(PlanError::TooManySections {
count: self.sections.len() + 1,
});
}
for prior in &self.sections {
if prior.kind == kind {
return Err(PlanError::DuplicateKind { kind });
}
}
self.sections.push(OwnedSection {
kind,
version,
alignment_log2,
payload,
});
Ok(self)
}
pub fn add_section_typed<T>(
&mut self,
kind: u32,
version: u32,
payload: &[T],
) -> Result<&mut Self, PlanError>
where
T: zerocopy::IntoBytes + zerocopy::Immutable,
{
let alignment = core::mem::align_of::<T>();
let alignment_log2 = match u8::try_from(alignment.trailing_zeros()) {
Ok(value) => value,
Err(_error) => {
return Err(PlanError::AlignmentTooLarge {
alignment_log2: u8::MAX,
});
}
};
if alignment_log2 > MAX_ALIGNMENT_LOG2 {
return Err(PlanError::AlignmentTooLarge { alignment_log2 });
}
let bytes = payload.as_bytes().to_vec();
self.add_section(kind, version, alignment_log2, bytes)
}
pub fn add_section_little_endian<T>(
&mut self,
kind: u32,
version: u32,
payload: &[T],
) -> Result<&mut Self, PlanError>
where
T: zerocopy::IntoBytes + zerocopy::Immutable,
{
self.add_section_typed(kind, version, payload)
}
pub fn add_section_widths<W>(
&mut self,
kind: u32,
version: u32,
values: &[W],
) -> Result<&mut Self, PlanError>
where
W: SnapshotWidth,
{
let words = oxgraph_layout_util::slice_to_le(values);
self.add_section_little_endian(kind, version, &words)
}
#[must_use]
pub const fn section_count(&self) -> usize {
self.sections.len()
}
pub fn finish(self) -> Result<Vec<u8>, PlanError> {
let pending: Vec<PendingSection<'_>> = self
.sections
.iter()
.map(|section| PendingSection {
kind: section.kind,
version: section.version,
alignment_log2: section.alignment_log2,
payload: section.payload.as_slice(),
})
.collect();
let plan = SnapshotPlan::new(&pending)?;
let needed = plan.encoded_len()?;
let mut out = vec![0u8; needed];
plan.write_into(&mut out)?;
Ok(out)
}
}