use std::fs::{self, File, OpenOptions};
use std::io::{Read, Seek, SeekFrom, Write};
use std::path::{Path, PathBuf};
use crate::error::{Neo4jStoreError, Result};
const CHECKPOINT_PREFIX: &str = "checkpoint.";
const TRANSACTION_LOG_PREFIX: &str = "neostore.transaction.db.";
const LOG_HEADER_SIZE: usize = 128;
const CHECKPOINT_RECORD_SIZE: usize = 232;
const DETACHED_CHECKPOINT_TYPE: u8 = 9;
const MIN_SUPPORTED_KERNEL_VERSION: u8 = 5;
const MAX_SUPPORTED_KERNEL_VERSION: u8 = 30;
pub(super) fn initialize_empty(database_directory: &Path) -> Result<()> {
let transaction_directory = expected_transaction_directory(database_directory)?;
fs::create_dir_all(&transaction_directory).map_err(|source| Neo4jStoreError::Write {
path: transaction_directory.clone(),
source,
})?;
let log_path = transaction_directory.join("neostore.transaction.db.0");
write_new(&log_path, &[0_u8; LOG_HEADER_SIZE])?;
let checkpoint_path = transaction_directory.join("checkpoint.0");
let mut checkpoint = vec![0_u8; LOG_HEADER_SIZE + CHECKPOINT_RECORD_SIZE];
let record = &mut checkpoint[LOG_HEADER_SIZE..];
record[0] = 30;
record[1] = DETACHED_CHECKPOINT_TYPE;
record[2..10].copy_from_slice(&0_u64.to_le_bytes());
record[10..18].copy_from_slice(&(LOG_HEADER_SIZE as u64).to_le_bytes());
let checksum = crc32c(&record[..CHECKPOINT_RECORD_SIZE - 4]);
record[CHECKPOINT_RECORD_SIZE - 4..].copy_from_slice(&checksum.to_le_bytes());
write_new(&checkpoint_path, &checkpoint)
}
pub(super) fn validate_initialization_path(database_directory: &Path) -> Result<()> {
expected_transaction_directory(database_directory).map(|_| ())
}
fn expected_transaction_directory(database_directory: &Path) -> Result<PathBuf> {
let database_name = database_directory
.file_name()
.ok_or_else(|| unsupported(database_directory, "database path has no database name"))?;
if let Some(databases_directory) = database_directory
.parent()
.filter(|parent| parent.file_name().and_then(|name| name.to_str()) == Some("databases"))
{
let data_directory = databases_directory.parent().ok_or_else(|| {
unsupported(database_directory, "database path has no data directory")
})?;
return Ok(data_directory.join("transactions").join(database_name));
}
Ok(database_directory.join(".littlegraph-transactions"))
}
fn write_new(path: &Path, bytes: &[u8]) -> Result<()> {
let mut file = OpenOptions::new()
.write(true)
.create_new(true)
.open(path)
.map_err(|source| Neo4jStoreError::Write {
path: path.to_path_buf(),
source,
})?;
file.write_all(bytes)
.and_then(|_| file.sync_all())
.map_err(|source| {
Neo4jStoreError::Write {
path: path.to_path_buf(),
source,
}
.into()
})
}
pub(super) fn validate_clean_shutdown(database_directory: &Path) -> Result<()> {
let transaction_directory = transaction_directory(database_directory)?;
let (checkpoint_version, checkpoint_path) = latest_versioned_file(
&transaction_directory,
CHECKPOINT_PREFIX,
database_directory,
)?
.ok_or_else(|| unsupported(database_directory, "no checkpoint log was found"))?;
let mut checkpoint_file =
File::open(&checkpoint_path).map_err(|source| Neo4jStoreError::Read {
path: checkpoint_path.clone(),
source,
})?;
let checkpoint_len = checkpoint_file
.metadata()
.map_err(|source| Neo4jStoreError::Read {
path: checkpoint_path.clone(),
source,
})?
.len();
if checkpoint_len < LOG_HEADER_SIZE as u64 {
return Err(unsupported(
database_directory,
format!(
"checkpoint log {} has a truncated header",
checkpoint_path.display()
),
));
}
let body_len = checkpoint_len - LOG_HEADER_SIZE as u64;
if body_len == 0 || !body_len.is_multiple_of(CHECKPOINT_RECORD_SIZE as u64) {
return Err(unsupported(
database_directory,
format!(
"checkpoint log {} does not end on a complete checkpoint record",
checkpoint_path.display()
),
));
}
let record_offset = checkpoint_len - CHECKPOINT_RECORD_SIZE as u64;
let mut record = [0_u8; CHECKPOINT_RECORD_SIZE];
checkpoint_file
.seek(SeekFrom::Start(record_offset))
.and_then(|_| checkpoint_file.read_exact(&mut record))
.map_err(|source| Neo4jStoreError::Read {
path: checkpoint_path.clone(),
source,
})?;
let record = &record;
if !(MIN_SUPPORTED_KERNEL_VERSION..=MAX_SUPPORTED_KERNEL_VERSION).contains(&record[0]) {
return Err(unsupported(
database_directory,
format!(
"checkpoint log {} uses unsupported kernel version {}",
checkpoint_path.display(),
record[0]
),
));
}
if record[1] != DETACHED_CHECKPOINT_TYPE {
return Err(unsupported(
database_directory,
format!(
"checkpoint log {} uses unsupported entry type {}",
checkpoint_path.display(),
record[1]
),
));
}
let expected_checksum = u32::from_le_bytes(
record[CHECKPOINT_RECORD_SIZE - 4..]
.try_into()
.expect("checkpoint checksum has fixed size"),
);
let actual_checksum = crc32c(&record[..CHECKPOINT_RECORD_SIZE - 4]);
if actual_checksum != expected_checksum {
return Err(unsupported(
database_directory,
format!(
"checkpoint log {} has an invalid checksum",
checkpoint_path.display()
),
));
}
let log_version = u64::from_le_bytes(record[2..10].try_into().expect("fixed log version"));
let log_offset = u64::from_le_bytes(record[10..18].try_into().expect("fixed log offset"));
if log_offset < LOG_HEADER_SIZE as u64 {
return Err(unsupported(
database_directory,
format!(
"checkpoint points to byte {log_offset} of transaction log {log_version}, before the {LOG_HEADER_SIZE}-byte log header ends"
),
));
}
let (highest_log_version, highest_log_path) = latest_versioned_file(
&transaction_directory,
TRANSACTION_LOG_PREFIX,
database_directory,
)?
.ok_or_else(|| unsupported(database_directory, "no transaction log was found"))?;
if highest_log_version != log_version {
return Err(unsupported(
database_directory,
format!(
"checkpoint {} points to transaction log {log_version}, but highest log version is {highest_log_version}",
checkpoint_version
),
));
}
let log_len = fs::metadata(&highest_log_path)
.map_err(|source| Neo4jStoreError::Read {
path: highest_log_path.clone(),
source,
})?
.len();
if log_len != log_offset {
return Err(unsupported(
database_directory,
format!(
"checkpoint points to byte {log_offset} of transaction log {log_version}, whose length is {log_len}; recovery may be required"
),
));
}
Ok(())
}
fn transaction_directory(database_directory: &Path) -> Result<PathBuf> {
let path = expected_transaction_directory(database_directory)?;
if !path.is_dir() {
return Err(unsupported(
database_directory,
format!("transaction log directory is missing: {}", path.display()),
));
}
Ok(path)
}
fn latest_versioned_file(
directory: &Path,
prefix: &str,
database_directory: &Path,
) -> Result<Option<(u64, PathBuf)>> {
let entries = fs::read_dir(directory).map_err(|source| Neo4jStoreError::Read {
path: directory.to_path_buf(),
source,
})?;
let mut latest = None;
for entry in entries {
let entry = entry.map_err(|source| Neo4jStoreError::Read {
path: directory.to_path_buf(),
source,
})?;
let name = entry.file_name();
let Some(version) = name
.to_str()
.and_then(|name| name.strip_prefix(prefix))
.and_then(|version| version.parse::<u64>().ok())
else {
continue;
};
let file_type = entry.file_type().map_err(|source| Neo4jStoreError::Read {
path: entry.path(),
source,
})?;
if !file_type.is_file() {
return Err(unsupported(
database_directory,
format!("log entry is not a file: {}", entry.path().display()),
));
}
if latest
.as_ref()
.is_none_or(|(latest_version, _)| version > *latest_version)
{
latest = Some((version, entry.path()));
}
}
Ok(latest)
}
fn crc32c(bytes: &[u8]) -> u32 {
let mut crc = !0_u32;
for byte in bytes {
crc ^= *byte as u32;
for _ in 0..8 {
crc = (crc >> 1) ^ (0x82f6_3b78 & (0_u32.wrapping_sub(crc & 1)));
}
}
!crc
}
fn unsupported(path: &Path, message: impl Into<String>) -> crate::Error {
Neo4jStoreError::UnsupportedWritableStore {
path: path.to_path_buf(),
message: message.into(),
}
.into()
}
#[cfg(test)]
mod tests {
use super::*;
fn database(log_offset: u64) -> (tempfile::TempDir, PathBuf) {
let home = tempfile::tempdir().unwrap();
let database = home.path().join("data/databases/neo4j");
let transactions = home.path().join("data/transactions/neo4j");
fs::create_dir_all(&database).unwrap();
fs::create_dir_all(&transactions).unwrap();
fs::write(
transactions.join("neostore.transaction.db.0"),
vec![0_u8; log_offset as usize],
)
.unwrap();
let mut checkpoint = vec![0_u8; LOG_HEADER_SIZE + CHECKPOINT_RECORD_SIZE];
let record = &mut checkpoint[LOG_HEADER_SIZE..];
record[0] = 30;
record[1] = DETACHED_CHECKPOINT_TYPE;
record[2..10].copy_from_slice(&0_u64.to_le_bytes());
record[10..18].copy_from_slice(&log_offset.to_le_bytes());
let checksum = crc32c(&record[..CHECKPOINT_RECORD_SIZE - 4]);
record[CHECKPOINT_RECORD_SIZE - 4..].copy_from_slice(&checksum.to_le_bytes());
fs::write(transactions.join("checkpoint.0"), checkpoint).unwrap();
(home, database)
}
#[test]
fn crc32c_matches_the_standard_test_vector() {
assert_eq!(crc32c(b"123456789"), 0xe306_9283);
}
#[test]
fn accepts_a_checkpoint_at_the_transaction_log_tail() {
let (_home, database) = database(256);
validate_clean_shutdown(&database).unwrap();
}
#[test]
fn validates_the_last_record_without_loading_a_sparse_checkpoint_log() {
let (home, database) = database(256);
let path = home.path().join("data/transactions/neo4j/checkpoint.0");
let bytes = fs::read(&path).unwrap();
let record: [u8; CHECKPOINT_RECORD_SIZE] = bytes[LOG_HEADER_SIZE..]
.try_into()
.expect("test checkpoint contains one record");
let record_count = (1_u64 << 40) / CHECKPOINT_RECORD_SIZE as u64;
let checkpoint_len = LOG_HEADER_SIZE as u64 + record_count * CHECKPOINT_RECORD_SIZE as u64;
let mut file = OpenOptions::new().write(true).open(&path).unwrap();
file.set_len(checkpoint_len).unwrap();
file.seek(SeekFrom::Start(
checkpoint_len - CHECKPOINT_RECORD_SIZE as u64,
))
.unwrap();
file.write_all(&record).unwrap();
validate_clean_shutdown(&database).unwrap();
}
#[test]
fn rejects_transactions_after_the_checkpoint() {
let (home, database) = database(256);
fs::write(
home.path()
.join("data/transactions/neo4j/neostore.transaction.db.0"),
vec![0_u8; 257],
)
.unwrap();
let error = validate_clean_shutdown(&database).expect_err("recovery must be required");
assert!(matches!(
error,
crate::Error::Neo4jStore(Neo4jStoreError::UnsupportedWritableStore {
message,
..
}) if message.contains("recovery may be required")
));
}
#[test]
fn rejects_a_checkpoint_before_the_transaction_log_header_end() {
let (_home, database) = database(0);
let error = validate_clean_shutdown(&database).expect_err("log header must be present");
assert!(error
.to_string()
.contains("before the 128-byte log header ends"));
}
#[test]
fn rejects_unsupported_checkpoint_kernel_versions() {
for kernel_version in [
MIN_SUPPORTED_KERNEL_VERSION - 1,
MAX_SUPPORTED_KERNEL_VERSION + 1,
] {
let (home, database) = database(256);
let path = home.path().join("data/transactions/neo4j/checkpoint.0");
let mut bytes = fs::read(&path).unwrap();
let record = &mut bytes[LOG_HEADER_SIZE..];
record[0] = kernel_version;
let checksum = crc32c(&record[..CHECKPOINT_RECORD_SIZE - 4]);
record[CHECKPOINT_RECORD_SIZE - 4..].copy_from_slice(&checksum.to_le_bytes());
fs::write(path, bytes).unwrap();
let error = validate_clean_shutdown(&database)
.expect_err("unsupported kernel version must be rejected");
assert!(error.to_string().contains("unsupported kernel version"));
}
}
#[test]
fn rejects_a_corrupt_checkpoint() {
let (home, database) = database(256);
let path = home.path().join("data/transactions/neo4j/checkpoint.0");
let mut bytes = fs::read(&path).unwrap();
bytes[LOG_HEADER_SIZE + 20] ^= 1;
fs::write(path, bytes).unwrap();
let error = validate_clean_shutdown(&database).expect_err("checksum must be checked");
assert!(error.to_string().contains("invalid checksum"));
}
#[test]
#[ignore = "requires NEO4J_REAL_FIXTURE generated by tools/neo4j-fixture-generator"]
fn validates_a_real_neo4j_shutdown_checkpoint() {
let database = std::env::var("NEO4J_REAL_FIXTURE").unwrap();
validate_clean_shutdown(Path::new(&database)).unwrap();
}
}