use std::fs::{File, OpenOptions};
use std::io::{Read, Seek, SeekFrom, Write};
use std::path::{Path, PathBuf};
use crate::error::{Neo4jStoreError, Result};
use super::Neo4jStoreFormat;
pub(super) const PAGE_SIZE: usize = 8192;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(super) struct PageLayout {
file_page_size: usize,
reserved_bytes: usize,
record_size: usize,
records_per_page: usize,
}
impl PageLayout {
pub(super) fn new(path: &Path, format: Neo4jStoreFormat, record_size: usize) -> Result<Self> {
Self::with_page_size(path, format, PAGE_SIZE, 0, record_size)
}
fn with_page_size(
path: &Path,
format: Neo4jStoreFormat,
page_size: usize,
reserved_bytes: usize,
record_size: usize,
) -> Result<Self> {
if page_size == 0 {
return Err(invalid_layout(path, "page size must be greater than zero"));
}
if record_size == 0 {
return Err(invalid_layout(
path,
"record size must be greater than zero",
));
}
let file_page_size = match format {
Neo4jStoreFormat::StandardV5_0 => page_size - page_size % record_size,
Neo4jStoreFormat::AlignedV5_0 => page_size,
};
if reserved_bytes >= file_page_size {
return Err(invalid_layout(
path,
format!(
"reserved bytes {reserved_bytes} must be smaller than file page size {file_page_size}"
),
));
}
let records_per_page = (file_page_size - reserved_bytes) / record_size;
if records_per_page == 0 {
return Err(invalid_layout(
path,
format!(
"record size {record_size} does not fit in page payload of {} bytes",
file_page_size - reserved_bytes
),
));
}
Ok(Self {
file_page_size,
reserved_bytes,
record_size,
records_per_page,
})
}
pub(super) fn record_size(self) -> usize {
self.record_size
}
pub(super) fn offset(self, record_id: u64, path: &Path) -> Result<u64> {
let records_per_page = self.records_per_page as u64;
let page_id = record_id / records_per_page;
let record_in_page = record_id % records_per_page;
let page_offset = page_id
.checked_mul(self.file_page_size as u64)
.ok_or_else(|| invalid_layout(path, "record page offset overflow"))?;
let record_offset = record_in_page
.checked_mul(self.record_size as u64)
.ok_or_else(|| invalid_layout(path, "record offset overflow"))?;
page_offset
.checked_add(self.reserved_bytes as u64)
.and_then(|offset| offset.checked_add(record_offset))
.ok_or_else(|| invalid_layout(path, "record file offset overflow"))
}
pub(super) fn record_count(self, file_size: u64) -> u64 {
let file_page_size = self.file_page_size as u64;
let complete_pages = file_size / file_page_size;
let trailing_bytes = file_size % file_page_size;
let trailing_payload = trailing_bytes.saturating_sub(self.reserved_bytes as u64);
let trailing_records =
(trailing_payload / self.record_size as u64).min(self.records_per_page as u64);
complete_pages * self.records_per_page as u64 + trailing_records
}
}
pub(super) fn validate_store_file(
path: &Path,
record_type: &str,
format: Neo4jStoreFormat,
record_size: usize,
) -> Result<()> {
let layout = PageLayout::new(path, format, record_size)?;
let mut reader = RecordReader::open(path, record_type, layout)?;
if let Some(last_record_id) = reader.record_count().checked_sub(1) {
reader.read(last_record_id)?;
}
Ok(())
}
pub(super) struct RecordReader {
file: File,
path: PathBuf,
record_type: String,
file_size: u64,
layout: PageLayout,
}
impl RecordReader {
pub(super) fn open(
path: impl AsRef<Path>,
record_type: impl Into<String>,
layout: PageLayout,
) -> Result<Self> {
let path = path.as_ref();
let file = File::open(path).map_err(|source| Neo4jStoreError::Read {
path: path.to_path_buf(),
source,
})?;
let file_size = file
.metadata()
.map_err(|source| Neo4jStoreError::Read {
path: path.to_path_buf(),
source,
})?
.len();
Ok(Self {
file,
path: path.to_path_buf(),
record_type: record_type.into(),
file_size,
layout,
})
}
pub(super) fn record_count(&self) -> u64 {
self.layout.record_count(self.file_size)
}
pub(super) fn offset(&self, record_id: u64) -> Result<u64> {
self.layout.offset(record_id, &self.path)
}
pub(super) fn path(&self) -> &Path {
&self.path
}
pub(super) fn read(&mut self, record_id: u64) -> Result<Vec<u8>> {
let offset = self.layout.offset(record_id, &self.path)?;
let end = offset
.checked_add(self.layout.record_size() as u64)
.ok_or_else(|| invalid_layout(&self.path, "record end offset overflow"))?;
if end > self.file_size {
return Err(Neo4jStoreError::RecordOutOfBounds {
path: self.path.clone(),
record_type: self.record_type.clone(),
record_id,
offset,
record_size: self.layout.record_size(),
file_size: self.file_size,
}
.into());
}
self.file
.seek(SeekFrom::Start(offset))
.map_err(|source| self.read_error(record_id, offset, source))?;
let mut bytes = vec![0_u8; self.layout.record_size()];
self.file
.read_exact(&mut bytes)
.map_err(|source| self.read_error(record_id, offset, source))?;
Ok(bytes)
}
fn read_error(&self, record_id: u64, offset: u64, source: std::io::Error) -> crate::Error {
Neo4jStoreError::RecordRead {
path: self.path.clone(),
record_type: self.record_type.clone(),
record_id,
offset,
source,
}
.into()
}
}
#[allow(dead_code)]
pub(super) struct RecordWriter {
file: File,
path: PathBuf,
record_type: String,
file_size: u64,
layout: PageLayout,
}
#[allow(dead_code)]
impl RecordWriter {
pub(super) fn open(
path: impl AsRef<Path>,
record_type: impl Into<String>,
layout: PageLayout,
) -> Result<Self> {
let path = path.as_ref();
let file = OpenOptions::new()
.read(true)
.write(true)
.open(path)
.map_err(|source| Neo4jStoreError::Write {
path: path.to_path_buf(),
source,
})?;
let file_size = file
.metadata()
.map_err(|source| Neo4jStoreError::Write {
path: path.to_path_buf(),
source,
})?
.len();
Ok(Self {
file,
path: path.to_path_buf(),
record_type: record_type.into(),
file_size,
layout,
})
}
pub(super) fn file_len(&self) -> u64 {
self.file_size
}
pub(super) fn record_count(&self) -> u64 {
self.layout.record_count(self.file_size)
}
pub(super) fn write(&mut self, record_id: u64, bytes: &[u8]) -> Result<()> {
if bytes.len() != self.layout.record_size() {
return Err(self.invalid(
record_id,
format!(
"record has {} bytes, expected {}",
bytes.len(),
self.layout.record_size()
),
));
}
let record_count = self.record_count();
if record_id > record_count {
return Err(self.invalid(
record_id,
format!(
"cannot skip record IDs while appending: current high ID is {record_count}"
),
));
}
let offset = self.layout.offset(record_id, &self.path)?;
self.file
.seek(SeekFrom::Start(offset))
.map_err(|source| self.write_error(record_id, offset, source))?;
self.file
.write_all(bytes)
.map_err(|source| self.write_error(record_id, offset, source))?;
let end = offset
.checked_add(bytes.len() as u64)
.ok_or_else(|| invalid_layout(&self.path, "record end offset overflow"))?;
self.file_size = self.file_size.max(end);
Ok(())
}
pub(super) fn restore_len(&mut self, file_size: u64) -> Result<()> {
self.file
.set_len(file_size)
.map_err(|source| Neo4jStoreError::Write {
path: self.path.clone(),
source,
})?;
self.file_size = file_size;
Ok(())
}
pub(super) fn sync_all(&self) -> Result<()> {
self.file
.sync_all()
.map_err(|source| Neo4jStoreError::Write {
path: self.path.clone(),
source,
})?;
Ok(())
}
fn invalid(&self, record_id: u64, message: impl Into<String>) -> crate::Error {
Neo4jStoreError::InvalidRecord {
path: self.path.clone(),
record_type: self.record_type.clone(),
record_id,
offset: self
.layout
.offset(record_id, &self.path)
.unwrap_or(u64::MAX),
context: "record write".to_owned(),
message: message.into(),
}
.into()
}
fn write_error(&self, record_id: u64, offset: u64, source: std::io::Error) -> crate::Error {
Neo4jStoreError::RecordWrite {
path: self.path.clone(),
record_type: self.record_type.clone(),
record_id,
offset,
source,
}
.into()
}
}
fn invalid_layout(path: &Path, message: impl Into<String>) -> crate::Error {
Neo4jStoreError::InvalidPageLayout {
path: path.to_path_buf(),
message: message.into(),
}
.into()
}
#[cfg(test)]
mod tests {
use std::io::Write;
use super::*;
fn layout(format: Neo4jStoreFormat, reserved_bytes: usize) -> PageLayout {
PageLayout::with_page_size(Path::new("fixture"), format, 8, reserved_bytes, 3).unwrap()
}
#[test]
fn standard_uses_record_aligned_file_pages_without_padding() {
let layout = layout(Neo4jStoreFormat::StandardV5_0, 0);
assert_eq!(layout.file_page_size, 6);
assert_eq!(layout.records_per_page, 2);
assert_eq!(layout.offset(0, Path::new("fixture")).unwrap(), 0);
assert_eq!(layout.offset(1, Path::new("fixture")).unwrap(), 3);
assert_eq!(layout.offset(2, Path::new("fixture")).unwrap(), 6);
assert_eq!(layout.offset(3, Path::new("fixture")).unwrap(), 9);
}
#[test]
fn aligned_skips_page_end_padding() {
let layout = layout(Neo4jStoreFormat::AlignedV5_0, 0);
assert_eq!(layout.file_page_size, 8);
assert_eq!(layout.records_per_page, 2);
assert_eq!(layout.offset(0, Path::new("fixture")).unwrap(), 0);
assert_eq!(layout.offset(1, Path::new("fixture")).unwrap(), 3);
assert_eq!(layout.offset(2, Path::new("fixture")).unwrap(), 8);
assert_eq!(layout.offset(3, Path::new("fixture")).unwrap(), 11);
}
#[test]
fn reserved_bytes_are_skipped_on_every_page() {
let layout = layout(Neo4jStoreFormat::AlignedV5_0, 2);
assert_eq!(layout.records_per_page, 2);
assert_eq!(layout.offset(0, Path::new("fixture")).unwrap(), 2);
assert_eq!(layout.offset(1, Path::new("fixture")).unwrap(), 5);
assert_eq!(layout.offset(2, Path::new("fixture")).unwrap(), 10);
}
#[test]
fn equivalent_standard_and_aligned_files_return_the_same_records() {
let directory = tempfile::tempdir().unwrap();
let standard_path = directory.path().join("standard.store");
let aligned_path = directory.path().join("aligned.store");
let records = [[1, 2, 3], [4, 5, 6], [7, 8, 9], [10, 11, 12]];
let mut standard_file = File::create(&standard_path).unwrap();
for record in records {
standard_file.write_all(&record).unwrap();
}
let mut aligned_file = File::create(&aligned_path).unwrap();
for page in records.chunks(2) {
for record in page {
aligned_file.write_all(record).unwrap();
}
aligned_file.write_all(&[0xee; 2]).unwrap();
}
let mut standard = RecordReader::open(
&standard_path,
"test",
layout(Neo4jStoreFormat::StandardV5_0, 0),
)
.unwrap();
let mut aligned = RecordReader::open(
&aligned_path,
"test",
layout(Neo4jStoreFormat::AlignedV5_0, 0),
)
.unwrap();
assert_eq!(standard.record_count(), 4);
assert_eq!(aligned.record_count(), 4);
for (record_id, expected) in records.into_iter().enumerate() {
assert_eq!(standard.read(record_id as u64).unwrap(), expected);
assert_eq!(aligned.read(record_id as u64).unwrap(), expected);
}
}
#[test]
fn rejects_invalid_layouts_and_out_of_bounds_records() {
let path = Path::new("fixture");
assert!(PageLayout::with_page_size(path, Neo4jStoreFormat::AlignedV5_0, 8, 0, 0).is_err());
assert!(PageLayout::with_page_size(path, Neo4jStoreFormat::AlignedV5_0, 8, 8, 3).is_err());
assert!(PageLayout::with_page_size(path, Neo4jStoreFormat::StandardV5_0, 8, 0, 9).is_err());
let directory = tempfile::tempdir().unwrap();
let store_path = directory.path().join("short.store");
std::fs::write(&store_path, [1_u8, 2]).unwrap();
let mut reader = RecordReader::open(
&store_path,
"node",
layout(Neo4jStoreFormat::AlignedV5_0, 0),
)
.unwrap();
let error = reader.read(0).unwrap_err();
assert!(matches!(
error,
crate::Error::Neo4jStore(Neo4jStoreError::RecordOutOfBounds { record_id: 0, .. })
));
}
#[test]
fn production_layout_uses_neo4j_page_size_and_no_reserved_bytes() {
let layout =
PageLayout::new(Path::new("nodes"), Neo4jStoreFormat::AlignedV5_0, 15).unwrap();
assert_eq!(layout.file_page_size, PAGE_SIZE);
assert_eq!(layout.reserved_bytes, 0);
assert_eq!(layout.records_per_page, 546);
}
#[test]
fn writer_overwrites_and_appends_standard_records() {
let directory = tempfile::tempdir().unwrap();
let path = directory.path().join("standard.store");
std::fs::write(&path, [1_u8, 2, 3, 4, 5, 6]).unwrap();
let mut writer =
RecordWriter::open(&path, "test", layout(Neo4jStoreFormat::StandardV5_0, 0)).unwrap();
let original_len = writer.file_len();
writer.write(1, &[7, 8, 9]).unwrap();
writer.write(2, &[10, 11, 12]).unwrap();
writer.sync_all().unwrap();
assert_eq!(
std::fs::read(&path).unwrap(),
[1, 2, 3, 7, 8, 9, 10, 11, 12]
);
writer.restore_len(original_len).unwrap();
assert_eq!(std::fs::read(&path).unwrap(), [1, 2, 3, 7, 8, 9]);
}
#[test]
fn writer_preserves_aligned_padding_and_appends_on_the_next_page() {
let directory = tempfile::tempdir().unwrap();
let path = directory.path().join("aligned.store");
std::fs::write(&path, [1_u8, 2, 3, 4, 5, 6, 0xee, 0xee]).unwrap();
let mut writer =
RecordWriter::open(&path, "test", layout(Neo4jStoreFormat::AlignedV5_0, 0)).unwrap();
writer.write(1, &[7, 8, 9]).unwrap();
writer.write(2, &[10, 11, 12]).unwrap();
assert_eq!(
std::fs::read(&path).unwrap(),
[1, 2, 3, 7, 8, 9, 0xee, 0xee, 10, 11, 12]
);
}
#[test]
fn writer_rejects_bad_record_sizes_and_skipped_ids() {
let directory = tempfile::tempdir().unwrap();
let path = directory.path().join("records.store");
std::fs::write(&path, [1_u8, 2, 3]).unwrap();
let mut writer =
RecordWriter::open(&path, "node", layout(Neo4jStoreFormat::StandardV5_0, 0)).unwrap();
let bad_size = writer.write(0, &[1, 2]).unwrap_err();
assert!(bad_size.to_string().contains("expected 3"));
let skipped = writer.write(2, &[4, 5, 6]).unwrap_err();
assert!(skipped.to_string().contains("current high ID is 1"));
assert_eq!(std::fs::read(&path).unwrap(), [1, 2, 3]);
}
}