use eyre::eyre;
use rkyv::api::high::HighDeserializer;
use rkyv::Archive;
use std::io::SeekFrom;
use tokio::fs::File;
use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt};
use super::SpaceInfoPage;
use crate::page::header::GeneralHeader;
use crate::page::ty::PageType;
use crate::page::PageId;
use crate::{
DataPage, GeneralPage, Link, Persistable, GENERAL_HEADER_SIZE, INNER_PAGE_SIZE, PAGE_SIZE,
};
#[derive(Debug)]
pub struct PageOverflowError {
pub page_id: PageId,
pub data_length: usize,
pub capacity: usize,
}
impl std::fmt::Display for PageOverflowError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
f,
"page {} write needs {} bytes, exceeding the {}-byte page slot",
self.page_id, self.data_length, self.capacity
)
}
}
impl std::error::Error for PageOverflowError {}
pub fn map_data_pages_to_general<const DATA_LENGTH: usize>(
pages: Vec<DataPage<DATA_LENGTH>>,
) -> Vec<GeneralPage<DataPage<DATA_LENGTH>>> {
let header = &mut GeneralHeader::new(1.into(), PageType::Data, 0.into());
let mut general_pages = vec![];
let mut pages = pages.into_iter();
if let Some(p) = pages.next() {
let general = GeneralPage {
header: *header,
inner: p,
};
general_pages.push(general);
}
let mut previous_header = header;
for p in pages {
let general = GeneralPage {
header: previous_header.follow_with(PageType::Data),
inner: p,
};
general_pages.push(general);
previous_header = &mut general_pages.last_mut().unwrap().header;
}
general_pages
}
pub async fn persist_page<'a, T>(
page: &'a mut GeneralPage<T>,
file: &'a mut File,
) -> eyre::Result<()>
where
T: Persistable + Send + Sync,
{
seek_to_page_start(file, page.header.page_id.0).await?;
let page_count = page.header.page_id.0 as i64 + 1;
persist_page_in_place(page, file).await?;
let curr_position = file.stream_position().await?;
file.seek(SeekFrom::Current(
(page_count * PAGE_SIZE as i64) - curr_position as i64,
))
.await?;
Ok(())
}
async fn persist_page_in_place<'a, T>(
page: &'a mut GeneralPage<T>,
file: &'a mut File,
) -> eyre::Result<()>
where
T: Persistable + Send + Sync,
{
let inner_bytes = page.inner.as_bytes();
let inner_length = inner_bytes.as_ref().len();
if inner_length > INNER_PAGE_SIZE {
return Err(eyre::Report::new(PageOverflowError {
page_id: page.header.page_id,
data_length: inner_length,
capacity: INNER_PAGE_SIZE,
}));
}
page.header.data_length = inner_length as u32;
file.write_all(page.header.as_bytes().as_ref()).await?;
file.write_all(inner_bytes.as_ref()).await?;
Ok(())
}
pub async fn persist_pages_batch<T>(pages: Vec<GeneralPage<T>>, file: &mut File) -> eyre::Result<()>
where
T: Persistable + Send + Sync,
{
let mut iter = pages.into_iter();
if let Some(mut page) = iter.next() {
seek_to_page_start(file, page.header.page_id.0).await?;
persist_page_in_place(&mut page, file).await?;
for mut page in iter {
seek_to_page_start_relatively(file, page.header.page_id.0).await?;
persist_page_in_place(&mut page, file).await?;
}
Ok(())
} else {
Ok(())
}
}
pub(crate) fn page_start_offset(index: u32) -> u64 {
index as u64 * PAGE_SIZE as u64
}
pub async fn seek_to_page_start(file: &mut File, index: u32) -> eyre::Result<()> {
file.seek(SeekFrom::Start(page_start_offset(index))).await?;
Ok(())
}
async fn seek_to_page_start_relatively(file: &mut File, index: u32) -> eyre::Result<()> {
let curr_position = file.stream_position().await?;
file.seek(SeekFrom::Current(
page_start_offset(index) as i64 - curr_position as i64,
))
.await?;
Ok(())
}
pub async fn seek_by_link(file: &mut File, link: Link) -> eyre::Result<()> {
file.seek(SeekFrom::Start(
link.page_id.0 as u64 * PAGE_SIZE as u64 + GENERAL_HEADER_SIZE as u64 + link.offset as u64,
))
.await?;
Ok(())
}
pub async fn update_at<const DATA_LENGTH: u32>(
file: &mut File,
link: Link,
new_data: &[u8],
) -> eyre::Result<()> {
if new_data.len() as u32 != link.length {
return Err(eyre!(
"New data length {} does not match link length {}",
new_data.len(),
link.length
));
}
if link.offset as u64 + link.length as u64 > DATA_LENGTH as u64 {
return Err(eyre!(
"Link range (offset: {}, length: {}) exceeds data bounds ({})",
link.offset,
link.length,
DATA_LENGTH
));
}
seek_by_link(file, link).await?;
file.write_all(new_data).await?;
Ok(())
}
pub async fn parse_general_header(file: &mut File) -> eyre::Result<GeneralHeader> {
let mut buffer = [0; GENERAL_HEADER_SIZE];
file.read_exact(&mut buffer).await?;
let archived = crate::access_archived::<<GeneralHeader as Archive>::Archived>(&buffer[..])
.map_err(|error| eyre::eyre!("torn or corrupt page header: {error}"))?;
let header = rkyv::deserialize::<_, rkyv::rancor::Error>(archived)
.map_err(|error| eyre::eyre!("page header failed to deserialize: {error}"))?;
Ok(header)
}
pub async fn parse_page<Page, const INNER_PAGE_SIZE: u32>(
file: &mut File,
index: u32,
) -> eyre::Result<GeneralPage<Page>>
where
Page: rkyv::Archive + Persistable,
<Page as rkyv::Archive>::Archived:
rkyv::Deserialize<Page, HighDeserializer<rkyv::rancor::Error>>,
{
seek_to_page_start(file, index).await?;
parse_page_in_place::<Page, INNER_PAGE_SIZE>(file).await
}
async fn parse_page_in_place<Page, const INNER_PAGE_SIZE: u32>(
file: &mut File,
) -> eyre::Result<GeneralPage<Page>>
where
Page: rkyv::Archive + Persistable,
<Page as rkyv::Archive>::Archived:
rkyv::Deserialize<Page, HighDeserializer<rkyv::rancor::Error>>,
{
let header = parse_general_header(file).await?;
let length = if header.data_length == 0 {
INNER_PAGE_SIZE
} else {
header.data_length
};
let mut buffer: Vec<u8> = vec![0u8; length as usize];
file.read_exact(&mut buffer).await?;
let info = Page::from_bytes(buffer.as_ref(), header.data_version);
Ok(GeneralPage {
header,
inner: info,
})
}
pub async fn parse_pages_batch<Page, const PAGE_SIZE: u32>(
file: &mut File,
indexes: Vec<u32>,
) -> eyre::Result<Vec<GeneralPage<Page>>>
where
Page: rkyv::Archive + Persistable,
<Page as rkyv::Archive>::Archived:
rkyv::Deserialize<Page, HighDeserializer<rkyv::rancor::Error>>,
{
let mut iter = indexes.into_iter();
if let Some(index) = iter.next() {
let mut pages = vec![];
seek_to_page_start(file, index).await?;
let page = parse_page_in_place::<Page, PAGE_SIZE>(file).await?;
pages.push(page);
for index in iter {
seek_to_page_start_relatively(file, index).await?;
let page = parse_page_in_place::<Page, PAGE_SIZE>(file).await?;
pages.push(page);
}
Ok(pages)
} else {
Ok(vec![])
}
}
pub async fn parse_general_header_by_index(
file: &mut File,
index: u32,
) -> eyre::Result<GeneralHeader> {
seek_to_page_start(file, index).await?;
let header = parse_general_header(file).await?;
Ok(header)
}
pub async fn parse_data_page<const PAGE_SIZE: u32, const INNER_PAGE_SIZE: usize>(
file: &mut File,
index: u32,
) -> eyre::Result<GeneralPage<DataPage<INNER_PAGE_SIZE>>> {
seek_to_page_start(file, index).await?;
parse_data_page_in_place::<PAGE_SIZE, INNER_PAGE_SIZE>(file).await
}
async fn parse_data_page_in_place<const PAGE_SIZE: u32, const INNER_PAGE_SIZE: usize>(
file: &mut File,
) -> eyre::Result<GeneralPage<DataPage<INNER_PAGE_SIZE>>> {
let header = parse_general_header(file).await?;
let mut buffer = [0u8; INNER_PAGE_SIZE];
if header.next_id == 0.into() {
#[allow(clippy::unused_io_amount)]
file.read(&mut buffer).await?;
} else {
file.read_exact(&mut buffer).await?;
}
let data = DataPage {
data: buffer,
length: header.data_length,
};
Ok(GeneralPage {
header,
inner: data,
})
}
pub async fn parse_data_pages_batch<const PAGE_SIZE: u32, const INNER_PAGE_SIZE: usize>(
file: &mut File,
indexes: Vec<u32>,
) -> eyre::Result<Vec<GeneralPage<DataPage<INNER_PAGE_SIZE>>>> {
let mut iter = indexes.into_iter();
if let Some(index) = iter.next() {
let mut pages = vec![];
seek_to_page_start(file, index).await?;
let page = parse_data_page_in_place::<PAGE_SIZE, INNER_PAGE_SIZE>(file).await?;
pages.push(page);
for index in iter {
seek_to_page_start_relatively(file, index).await?;
let page = parse_data_page_in_place::<PAGE_SIZE, INNER_PAGE_SIZE>(file).await?;
pages.push(page);
}
Ok(pages)
} else {
Ok(vec![])
}
}
pub async fn parse_space_info<const PAGE_SIZE: usize>(
file: &mut File,
) -> eyre::Result<SpaceInfoPage> {
file.seek(SeekFrom::Start(0)).await?;
let header = parse_general_header(file).await?;
let mut buffer = vec![0u8; header.data_length as usize];
file.read_exact(&mut buffer).await?;
Ok(SpaceInfoPage::from_bytes(&buffer, header.data_version))
}
#[cfg(test)]
mod tests {
use super::{page_start_offset, parse_data_pages_batch, persist_pages_batch};
use crate::page::header::GeneralHeader;
use crate::page::ty::PageType;
use crate::{DataPage, GeneralPage, DATA_VERSION, INNER_PAGE_SIZE, PAGE_SIZE};
const FIRST_PAGE_PAST_4_GIB: u32 = (u32::MAX / PAGE_SIZE as u32) + 1;
#[test]
fn page_start_offset_is_computed_in_u64() {
assert_eq!(
page_start_offset(FIRST_PAGE_PAST_4_GIB),
FIRST_PAGE_PAST_4_GIB as u64 * PAGE_SIZE as u64
);
assert!(page_start_offset(FIRST_PAGE_PAST_4_GIB) > u32::MAX as u64);
assert_eq!(
page_start_offset(u32::MAX),
u32::MAX as u64 * PAGE_SIZE as u64
);
assert_ne!(
page_start_offset(FIRST_PAGE_PAST_4_GIB),
FIRST_PAGE_PAST_4_GIB.wrapping_mul(PAGE_SIZE as u32) as u64
);
}
fn data_page_with_marker(marker: &[u8]) -> DataPage<INNER_PAGE_SIZE> {
let mut data = [0u8; INNER_PAGE_SIZE];
data[..marker.len()].copy_from_slice(marker);
DataPage {
length: marker.len() as u32,
data,
}
}
#[tokio::test]
async fn persist_page_rejects_inner_data_past_the_page_slot() {
const OVERSIZED: usize = crate::PAGE_SIZE + 128;
let path = std::env::temp_dir().join(format!(
"data_bucket_persist_overflow_{}.wt",
std::process::id()
));
let mut file = tokio::fs::OpenOptions::new()
.read(true)
.write(true)
.create(true)
.truncate(true)
.open(&path)
.await
.unwrap();
let mut page = GeneralPage {
header: GeneralHeader {
data_version: DATA_VERSION,
space_id: 1.into(),
page_id: 1.into(),
previous_id: 0.into(),
next_id: 0.into(),
page_type: PageType::Data,
data_length: 0,
},
inner: DataPage {
length: OVERSIZED as u32,
data: [7u8; OVERSIZED],
},
};
let err = super::persist_page(&mut page, &mut file).await.unwrap_err();
assert!(
err.downcast_ref::<super::PageOverflowError>().is_some(),
"expected PageOverflowError, got: {err}"
);
assert_eq!(file.metadata().await.unwrap().len(), 0);
page.inner.length = 64;
super::persist_page(&mut page, &mut file).await.unwrap();
drop(file);
std::fs::remove_file(&path).unwrap();
}
#[tokio::test]
async fn update_at_rejects_offset_plus_length_wrapping_u32() {
let path = std::env::temp_dir().join(format!(
"data_bucket_update_at_wrap_{}.wt",
std::process::id()
));
let mut file = tokio::fs::OpenOptions::new()
.read(true)
.write(true)
.create(true)
.truncate(true)
.open(&path)
.await
.unwrap();
let link = crate::Link {
page_id: 1.into(),
offset: u32::MAX - 2,
length: 8,
};
let err = super::update_at::<100>(&mut file, link, &[1, 2, 3, 4, 5, 6, 7, 8])
.await
.unwrap_err();
assert!(err.to_string().contains("exceeds data bounds"));
drop(file);
std::fs::remove_file(&path).unwrap();
}
#[tokio::test]
async fn batch_persist_and_parse_address_pages_past_4_gib() {
const FIRST_MARKER: &[u8] = b"FIRSTPG!";
const BOUNDARY_MARKER: &[u8] = b"BOUNDARY";
let path = std::env::temp_dir().join(format!(
"data_bucket_seek_past_4gib_{}.wt",
std::process::id()
));
let mut file = tokio::fs::OpenOptions::new()
.read(true)
.write(true)
.create(true)
.truncate(true)
.open(&path)
.await
.unwrap();
let first_page = GeneralPage {
header: GeneralHeader {
data_version: DATA_VERSION,
space_id: 1.into(),
page_id: 1.into(),
previous_id: 0.into(),
next_id: 2.into(),
page_type: PageType::Data,
data_length: 0,
},
inner: data_page_with_marker(FIRST_MARKER),
};
let boundary_page = GeneralPage {
header: GeneralHeader {
data_version: DATA_VERSION,
space_id: 1.into(),
page_id: FIRST_PAGE_PAST_4_GIB.into(),
previous_id: 1.into(),
next_id: 0.into(),
page_type: PageType::Data,
data_length: 0,
},
inner: data_page_with_marker(BOUNDARY_MARKER),
};
persist_pages_batch(vec![first_page, boundary_page], &mut file)
.await
.unwrap();
tokio::io::AsyncWriteExt::flush(&mut file).await.unwrap();
let file_length = file.metadata().await.unwrap().len();
assert!(file_length > page_start_offset(FIRST_PAGE_PAST_4_GIB));
let pages = parse_data_pages_batch::<{ PAGE_SIZE as u32 }, INNER_PAGE_SIZE>(
&mut file,
vec![1, FIRST_PAGE_PAST_4_GIB],
)
.await
.unwrap();
assert_eq!(pages.len(), 2);
assert_eq!(pages[0].header.page_id, 1.into());
assert_eq!(&pages[0].inner.data[..FIRST_MARKER.len()], FIRST_MARKER);
assert_eq!(pages[1].header.page_id, FIRST_PAGE_PAST_4_GIB.into());
assert_eq!(
&pages[1].inner.data[..BOUNDARY_MARKER.len()],
BOUNDARY_MARKER
);
drop(file);
std::fs::remove_file(&path).unwrap();
}
}