pub mod compression;
pub mod fault;
pub mod remote;
pub mod replication;
pub mod wal;
use crate::error::{Error, Result};
use std::collections::{HashMap, HashSet};
use std::fs::{File, OpenOptions};
use std::io::{Read, Seek, SeekFrom, Write};
use std::path::{Path, PathBuf};
use std::sync::Arc;
pub use compression::{COMPRESSED_PAGE_MAGIC, compress_page_frame, decompress_page_frame};
#[cfg(feature = "cloud-s3")]
pub use remote::CloudS3RemoteStorage;
pub use remote::{
MockRemoteRangeStorage, RemotePager, RemoteRangeReader, RemoteStorageAdapter, S3StorageConfig,
};
pub use replication::{
WalReplicationChunk, WalReplicationFrame, WalReplicationReceiver, WalReplicationStream,
};
pub use wal::{DEFAULT_CHECKPOINT_THRESHOLD, StorageHealth, Wal, WalFrameHeader, WalHeader};
pub const DEFAULT_PAGE_SIZE: u16 = 4096;
pub const DATABASE_MAGIC: [u8; 8] = *b"TAPIRUS\0";
pub const FILE_FORMAT_VERSION: u16 = 1;
pub const DATABASE_HEADER_SIZE: usize = 100;
pub type PageId = u32;
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct DatabaseHeader {
pub magic: [u8; 8],
pub page_size: u16,
pub version: u16,
pub change_counter: u32,
pub total_pages: u32,
pub freelist_trunk: u32,
pub freelist_count: u32,
pub schema_cookie: u32,
pub user_version: u32,
pub vector_index_page: u32,
pub wal_sequence: u32,
pub header_crc32: u32,
pub encryption_flags: u8,
pub salt: [u8; 16],
pub kcv: [u8; 16],
pub compression_flags: u8,
}
impl Default for DatabaseHeader {
fn default() -> Self {
let mut header = Self {
magic: DATABASE_MAGIC,
page_size: DEFAULT_PAGE_SIZE,
version: FILE_FORMAT_VERSION,
change_counter: 0,
total_pages: 1, freelist_trunk: 0,
freelist_count: 0,
schema_cookie: 0,
user_version: 0,
vector_index_page: 0,
wal_sequence: 0,
header_crc32: 0,
encryption_flags: 0,
salt: [0u8; 16],
kcv: [0u8; 16],
compression_flags: 0,
};
header.header_crc32 = header.calculate_crc();
header
}
}
impl DatabaseHeader {
pub fn new(page_size: u16) -> Self {
let mut header = Self {
page_size,
..Default::default()
};
header.header_crc32 = header.calculate_crc();
header
}
pub fn calculate_crc(&self) -> u32 {
let mut buf = [0u8; 44];
buf[0..8].copy_from_slice(&self.magic);
buf[8..10].copy_from_slice(&self.page_size.to_le_bytes());
buf[10..12].copy_from_slice(&self.version.to_le_bytes());
buf[12..16].copy_from_slice(&self.change_counter.to_le_bytes());
buf[16..20].copy_from_slice(&self.total_pages.to_le_bytes());
buf[20..24].copy_from_slice(&self.freelist_trunk.to_le_bytes());
buf[24..28].copy_from_slice(&self.freelist_count.to_le_bytes());
buf[28..32].copy_from_slice(&self.schema_cookie.to_le_bytes());
buf[32..36].copy_from_slice(&self.user_version.to_le_bytes());
buf[36..40].copy_from_slice(&self.vector_index_page.to_le_bytes());
buf[40..44].copy_from_slice(&self.wal_sequence.to_le_bytes());
let mut hasher = crc32fast::Hasher::new();
hasher.update(&buf);
hasher.finalize()
}
pub fn to_bytes(&self) -> [u8; DATABASE_HEADER_SIZE] {
let mut buf = [0u8; DATABASE_HEADER_SIZE];
buf[0..8].copy_from_slice(&self.magic);
buf[8..10].copy_from_slice(&self.page_size.to_le_bytes());
buf[10..12].copy_from_slice(&self.version.to_le_bytes());
buf[12..16].copy_from_slice(&self.change_counter.to_le_bytes());
buf[16..20].copy_from_slice(&self.total_pages.to_le_bytes());
buf[20..24].copy_from_slice(&self.freelist_trunk.to_le_bytes());
buf[24..28].copy_from_slice(&self.freelist_count.to_le_bytes());
buf[28..32].copy_from_slice(&self.schema_cookie.to_le_bytes());
buf[32..36].copy_from_slice(&self.user_version.to_le_bytes());
buf[36..40].copy_from_slice(&self.vector_index_page.to_le_bytes());
buf[40..44].copy_from_slice(&self.wal_sequence.to_le_bytes());
let crc = self.calculate_crc();
buf[44..48].copy_from_slice(&crc.to_le_bytes());
buf[48] = self.encryption_flags;
buf[49..65].copy_from_slice(&self.salt);
buf[65..81].copy_from_slice(&self.kcv);
buf[81] = self.compression_flags;
buf
}
pub fn from_bytes(slice: &[u8]) -> Result<Self> {
if slice.len() < DATABASE_HEADER_SIZE {
return Err(Error::Corrupted(
"Slice is smaller than 100-byte header".into(),
));
}
let mut magic = [0u8; 8];
magic.copy_from_slice(&slice[0..8]);
if magic != DATABASE_MAGIC {
return Err(Error::InvalidMagic);
}
let page_size = u16::from_le_bytes([slice[8], slice[9]]);
if !page_size.is_power_of_two() || page_size < 512 {
return Err(Error::InvalidPageSize(page_size));
}
let version = u16::from_le_bytes([slice[10], slice[11]]);
let change_counter = u32::from_le_bytes([slice[12], slice[13], slice[14], slice[15]]);
let total_pages = u32::from_le_bytes([slice[16], slice[17], slice[18], slice[19]]);
let freelist_trunk = u32::from_le_bytes([slice[20], slice[21], slice[22], slice[23]]);
let freelist_count = u32::from_le_bytes([slice[24], slice[25], slice[26], slice[27]]);
let schema_cookie = u32::from_le_bytes([slice[28], slice[29], slice[30], slice[31]]);
let user_version = u32::from_le_bytes([slice[32], slice[33], slice[34], slice[35]]);
let vector_index_page = u32::from_le_bytes([slice[36], slice[37], slice[38], slice[39]]);
let wal_sequence = u32::from_le_bytes([slice[40], slice[41], slice[42], slice[43]]);
let header_crc32 = u32::from_le_bytes([slice[44], slice[45], slice[46], slice[47]]);
let encryption_flags = slice[48];
let mut salt = [0u8; 16];
salt.copy_from_slice(&slice[49..65]);
let mut kcv = [0u8; 16];
kcv.copy_from_slice(&slice[65..81]);
let compression_flags = slice[81];
let header = Self {
magic,
page_size,
version,
change_counter,
total_pages,
freelist_trunk,
freelist_count,
schema_cookie,
user_version,
vector_index_page,
wal_sequence,
header_crc32,
encryption_flags,
salt,
kcv,
compression_flags,
};
let calculated_crc = header.calculate_crc();
if calculated_crc != header_crc32 {
return Err(Error::PageCorrupted(1, calculated_crc, header_crc32));
}
if header.version != FILE_FORMAT_VERSION {
return Err(Error::UnsupportedFormat(header.version));
}
Ok(header)
}
}
#[derive(Debug, Clone)]
struct TransactionSavepoint {
header: DatabaseHeader,
undo_pages: HashMap<PageId, Option<Vec<u8>>>,
dirty_pages: HashSet<PageId>,
total_pages: u32,
wal_frame_count: usize,
}
#[derive(Debug, Clone)]
pub struct NamedSavepoint {
header: DatabaseHeader,
pub name: String,
pub undo_pages: HashMap<PageId, Option<Vec<u8>>>,
pub dirty_pages: HashSet<PageId>,
pub total_pages: u32,
pub wal_frame_count: usize,
}
#[derive(Debug, Clone, Copy, Default)]
struct LruNode {
prev: Option<PageId>,
next: Option<PageId>,
}
pub struct Pager {
header: DatabaseHeader,
file: Option<File>,
_path: Option<PathBuf>,
_lock_file: Option<File>,
_lock_path: Option<PathBuf>,
in_memory_pages: HashMap<PageId, Vec<u8>>,
wal: Option<Wal>,
wal_overlay: HashMap<PageId, Arc<Vec<u8>>>,
concurrent_wal_path: Option<PathBuf>,
read_only: bool,
concurrent_access: bool,
busy_timeout: std::time::Duration,
synced_wal_fingerprint: (u64, u32, u32, u32),
synced_base_len: u64,
write_generation: u64,
checkpoint_threshold: usize,
cipher: Option<crate::crypto::DatabaseCipher>,
transaction_savepoint: Option<TransactionSavepoint>,
named_savepoints: Vec<NamedSavepoint>,
cache_capacity: usize,
lru_head: Option<PageId>,
lru_tail: Option<PageId>,
lru_nodes: HashMap<PageId, LruNode>,
remote_reader: Option<Arc<dyn RemoteRangeReader>>,
remote_writer: Option<Arc<dyn RemoteStorageAdapter>>,
}
#[cfg(not(windows))]
fn is_process_alive(pid: u32) -> bool {
if pid == std::process::id() {
return true;
}
#[cfg(unix)]
{
std::path::Path::new(&format!("/proc/{pid}")).exists()
}
#[cfg(not(unix))]
{
true
}
}
impl Pager {
pub(crate) fn ensure_io_ready(&self) -> Result<()> {
if self.wal.as_ref().is_some_and(Wal::requires_reopen) {
return Err(Error::TransactionError(
"Storage requires reopen after an incomplete I/O recovery".into(),
));
}
Ok(())
}
pub(crate) fn read_header_for_open(path: &Path) -> Result<Option<DatabaseHeader>> {
let wal_path = path.with_extension("tapir-wal");
let mut file = match File::open(path) {
Ok(file) => file,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
if wal_path
.metadata()
.is_ok_and(|meta| meta.len() > wal::WAL_HEADER_SIZE as u64)
{
return Err(Error::Corrupted(
"Base database is missing but an orphan WAL exists".into(),
));
}
return Ok(None);
}
Err(error) => return Err(error.into()),
};
let length = file.metadata()?.len();
let parsed = if length >= DATABASE_HEADER_SIZE as u64 {
let mut bytes = [0; DATABASE_HEADER_SIZE];
file.read_exact(&mut bytes)?;
DatabaseHeader::from_bytes(&bytes)
} else {
Err(Error::Corrupted(
"Database file is shorter than its header".into(),
))
};
match parsed {
Ok(header) => Ok(Some(header)),
Err(error @ Error::UnsupportedFormat(_)) => Err(error),
Err(original) => {
if wal_path.exists()
&& let Some(page) = Wal::read_committed_page(&wal_path, 1)?
{
return DatabaseHeader::from_bytes(&page).map(Some);
}
if length == 0 { Ok(None) } else { Err(original) }
}
}
}
pub fn open_with_config(path: &Path, page_size: u16, cache_capacity: usize) -> Result<Self> {
Self::open_with_cipher(path, page_size, cache_capacity, None)
}
fn acquire_lock_file(
db_path: &Path,
lock_path: &Path,
timeout: std::time::Duration,
) -> Result<File> {
let start = web_time::Instant::now();
loop {
let create_attempt = {
let mut opts = OpenOptions::new();
opts.read(true).write(true).create_new(true);
#[cfg(windows)]
{
use std::os::windows::fs::OpenOptionsExt;
opts.share_mode(0);
}
opts.open(lock_path)
};
match create_attempt {
Ok(mut lf) => {
let _ = write!(lf, "{}", std::process::id());
let _ = lf.flush();
return Ok(lf);
}
Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
#[cfg(windows)]
{
use std::os::windows::fs::OpenOptionsExt;
match OpenOptions::new()
.read(true)
.write(true)
.share_mode(0)
.open(lock_path)
{
Ok(mut lf) => {
let _ = lf.set_len(0);
let _ = lf.seek(SeekFrom::Start(0));
let _ = write!(lf, "{}", std::process::id());
let _ = lf.flush();
return Ok(lf);
}
Err(open_err)
if open_err.kind() == std::io::ErrorKind::PermissionDenied
|| matches!(open_err.raw_os_error(), Some(32 | 33)) =>
{
}
Err(open_err) if open_err.kind() == std::io::ErrorKind::NotFound => {
continue;
}
Err(open_err) => return Err(Error::Io(open_err)),
}
}
#[cfg(not(windows))]
{
let is_stale = if let Ok(content) = std::fs::read_to_string(lock_path) {
if let Ok(pid) = content.trim().parse::<u32>() {
!is_process_alive(pid)
} else {
false
}
} else {
false
};
if is_stale {
let _ = std::fs::remove_file(lock_path);
continue;
}
}
if start.elapsed() >= timeout {
return Err(Error::Busy(format!(
"Database file '{}' is locked by another process (active lockfile: {})",
db_path.display(),
lock_path.display()
)));
}
std::thread::sleep(std::time::Duration::from_millis(5));
}
Err(e)
if cfg!(windows)
&& e.kind() == std::io::ErrorKind::PermissionDenied
&& start.elapsed() < timeout =>
{
std::thread::sleep(std::time::Duration::from_millis(5));
}
Err(e) => return Err(Error::Io(e)),
}
}
}
pub fn open_with_cipher(
path: &Path,
page_size: u16,
_cache_capacity: usize,
cipher: Option<crate::crypto::DatabaseCipher>,
) -> Result<Self> {
let lock_path = path.with_extension("tapir-lock");
let lock_file = Self::acquire_lock_file(path, &lock_path, std::time::Duration::ZERO)?;
struct LockGuard {
file: Option<std::fs::File>,
path: std::path::PathBuf,
active: bool,
}
impl Drop for LockGuard {
fn drop(&mut self) {
if self.active {
self.file.take();
let _ = std::fs::remove_file(&self.path);
}
}
}
let mut _guard = LockGuard {
file: Some(lock_file),
path: lock_path.clone(),
active: true,
};
let stored_header = Self::read_header_for_open(path)?;
let mut file = OpenOptions::new()
.read(true)
.write(true)
.create(true)
.truncate(false)
.open(path)?;
let mut header = if let Some(h) = stored_header {
if h.encryption_flags == 1 {
if let Some(ref c) = cipher {
if !c.verify_kcv(&h.kcv) {
return Err(Error::DecryptionFailed(1));
}
} else {
return Err(Error::EncryptedDatabase);
}
}
h
} else {
let mut h = DatabaseHeader::new(page_size);
if let Some(ref c) = cipher {
h.encryption_flags = 1;
h.salt = *c.salt();
h.kcv = c.generate_kcv()?;
}
let mut page1 = vec![0u8; page_size as usize];
page1[0..DATABASE_HEADER_SIZE].copy_from_slice(&h.to_bytes());
let disk_page1 = if let Some(ref c) = cipher {
c.encrypt_page(1, &page1)?
} else {
page1
};
file.seek(SeekFrom::Start(0))?;
file.write_all(&disk_page1)?;
file.sync_data()?;
h
};
let mut in_memory_pages = HashMap::new();
let wal_path = path.with_extension("tapir-wal");
let mut wal = Wal::open(&wal_path, header.page_size as u32)?;
let mut file_opt = Some(file);
let mut recovered_pages = HashMap::new();
wal.recover(&mut file_opt, &mut recovered_pages)?;
for (pid, raw) in recovered_pages {
let pt = if let Some(ref c) = cipher {
c.decrypt_page(pid, &raw)?
} else {
raw
};
in_memory_pages.insert(pid, pt);
}
if let Some(f) = &mut file_opt {
f.seek(SeekFrom::Start(0))?;
let mut h_buf = [0u8; DATABASE_HEADER_SIZE];
f.read_exact(&mut h_buf)?;
header = DatabaseHeader::from_bytes(&h_buf)?;
}
let cache_capacity = if _cache_capacity == 0 {
256
} else {
_cache_capacity
};
_guard.active = false;
let lock_file = _guard.file.take();
let mut pager = Self {
header,
file: file_opt,
_path: Some(path.to_path_buf()),
_lock_file: lock_file,
_lock_path: Some(lock_path),
in_memory_pages: HashMap::new(),
wal: Some(wal),
wal_overlay: HashMap::new(),
concurrent_wal_path: None,
read_only: false,
concurrent_access: false,
busy_timeout: std::time::Duration::ZERO,
synced_wal_fingerprint: (0, 0, 0, 0),
synced_base_len: 0,
write_generation: 0,
checkpoint_threshold: DEFAULT_CHECKPOINT_THRESHOLD,
cipher,
transaction_savepoint: None,
named_savepoints: Vec::new(),
cache_capacity,
lru_head: None,
lru_tail: None,
lru_nodes: HashMap::new(),
remote_reader: None,
remote_writer: None,
};
for (pid, page) in in_memory_pages {
let page = pager.decompress_if_needed(pid, page)?;
pager.in_memory_pages.insert(pid, page);
pager.touch_lru(pid);
}
Ok(pager)
}
pub fn open_read_only_with_cipher(
path: &Path,
cache_capacity: usize,
cipher: Option<crate::crypto::DatabaseCipher>,
) -> Result<Self> {
let header = Self::read_header_for_open(path)?.ok_or_else(|| {
Error::Io(std::io::Error::new(
std::io::ErrorKind::NotFound,
format!("Database file '{}' does not exist", path.display()),
))
})?;
if header.encryption_flags == 1 {
if let Some(ref c) = cipher {
if !c.verify_kcv(&header.kcv) {
return Err(Error::DecryptionFailed(1));
}
} else {
return Err(Error::EncryptedDatabase);
}
}
let file = OpenOptions::new().read(true).open(path)?;
let cache_capacity = if cache_capacity == 0 {
256
} else {
cache_capacity
};
let wal_path = path.with_extension("tapir-wal");
let mut pager = Self {
header,
file: Some(file),
_path: Some(path.to_path_buf()),
_lock_file: None,
_lock_path: None,
in_memory_pages: HashMap::new(),
wal: None,
wal_overlay: HashMap::new(),
concurrent_wal_path: Some(wal_path),
read_only: true,
concurrent_access: false,
busy_timeout: std::time::Duration::ZERO,
synced_wal_fingerprint: (u64::MAX, 0, 0, 0),
synced_base_len: u64::MAX,
write_generation: 0,
checkpoint_threshold: DEFAULT_CHECKPOINT_THRESHOLD,
cipher,
transaction_savepoint: None,
named_savepoints: Vec::new(),
cache_capacity,
lru_head: None,
lru_tail: None,
lru_nodes: HashMap::new(),
remote_reader: None,
remote_writer: None,
};
pager.refresh_committed_state()?;
Ok(pager)
}
pub fn open_concurrent_with_cipher(
path: &Path,
page_size: u16,
cache_capacity: usize,
cipher: Option<crate::crypto::DatabaseCipher>,
busy_timeout: std::time::Duration,
) -> Result<Self> {
let lock_path = path.with_extension("tapir-lock");
let wal_path = path.with_extension("tapir-wal");
if !path.exists() {
let init_lock = Self::acquire_lock_file(path, &lock_path, busy_timeout)?;
struct InitGuard(Option<File>, PathBuf);
impl Drop for InitGuard {
fn drop(&mut self) {
self.0.take();
let _ = std::fs::remove_file(&self.1);
}
}
let _guard = InitGuard(Some(init_lock), lock_path.clone());
if !path.exists() {
let mut h = DatabaseHeader::new(page_size);
if let Some(ref c) = cipher {
h.encryption_flags = 1;
h.salt = *c.salt();
h.kcv = c.generate_kcv()?;
}
let mut page1 = vec![0u8; page_size as usize];
crate::btree::engine::init_leaf_page(&mut page1, 1);
page1[0..DATABASE_HEADER_SIZE].copy_from_slice(&h.to_bytes());
let disk_page1 = if let Some(ref c) = cipher {
c.encrypt_page(1, &page1)?
} else {
page1
};
let mut f = OpenOptions::new()
.read(true)
.write(true)
.create(true)
.truncate(false)
.open(path)?;
f.seek(SeekFrom::Start(0))?;
f.write_all(&disk_page1)?;
f.sync_data()?;
let _wal = Wal::open(&wal_path, h.page_size as u32)?;
}
}
let header = Self::read_header_for_open(path)?.ok_or_else(|| {
Error::Corrupted(format!("Failed to read header from '{}'", path.display()))
})?;
if header.encryption_flags == 1 {
if let Some(ref c) = cipher {
if !c.verify_kcv(&header.kcv) {
return Err(Error::DecryptionFailed(1));
}
} else {
return Err(Error::EncryptedDatabase);
}
}
let file = OpenOptions::new().read(true).write(true).open(path)?;
let cache_capacity = if cache_capacity == 0 {
256
} else {
cache_capacity
};
let mut pager = Self {
header,
file: Some(file),
_path: Some(path.to_path_buf()),
_lock_file: None,
_lock_path: Some(lock_path),
in_memory_pages: HashMap::new(),
wal: None,
wal_overlay: HashMap::new(),
concurrent_wal_path: Some(wal_path),
read_only: false,
concurrent_access: true,
busy_timeout,
synced_wal_fingerprint: (u64::MAX, 0, 0, 0),
synced_base_len: u64::MAX,
write_generation: 0,
checkpoint_threshold: DEFAULT_CHECKPOINT_THRESHOLD,
cipher,
transaction_savepoint: None,
named_savepoints: Vec::new(),
cache_capacity,
lru_head: None,
lru_tail: None,
lru_nodes: HashMap::new(),
remote_reader: None,
remote_writer: None,
};
pager.refresh_committed_state()?;
Ok(pager)
}
pub fn open_in_memory(page_size: u16, cache_capacity: usize) -> Result<Self> {
Self::open_in_memory_with_cipher(page_size, cache_capacity, None)
}
pub fn open_in_memory_with_cipher(
page_size: u16,
_cache_capacity: usize,
cipher: Option<crate::crypto::DatabaseCipher>,
) -> Result<Self> {
let mut header = DatabaseHeader::new(page_size);
if let Some(ref c) = cipher {
header.encryption_flags = 1;
header.salt = *c.salt();
header.kcv = c.generate_kcv()?;
}
let mut page1 = vec![0u8; page_size as usize];
page1[0..DATABASE_HEADER_SIZE].copy_from_slice(&header.to_bytes());
let mut in_memory_pages = HashMap::new();
in_memory_pages.insert(1, page1);
let wal = Wal::open_in_memory(page_size as u32);
let cache_capacity = if _cache_capacity == 0 {
256
} else {
_cache_capacity
};
let mut pager = Self {
header,
file: None,
_path: None,
_lock_file: None,
_lock_path: None,
in_memory_pages,
wal: Some(wal),
wal_overlay: HashMap::new(),
concurrent_wal_path: None,
read_only: false,
concurrent_access: false,
busy_timeout: std::time::Duration::ZERO,
synced_wal_fingerprint: (0, 0, 0, 0),
synced_base_len: 0,
write_generation: 0,
checkpoint_threshold: DEFAULT_CHECKPOINT_THRESHOLD,
cipher,
transaction_savepoint: None,
named_savepoints: Vec::new(),
cache_capacity,
lru_head: None,
lru_tail: None,
lru_nodes: HashMap::new(),
remote_reader: None,
remote_writer: None,
};
pager.touch_lru(1);
Ok(pager)
}
pub fn open_in_memory_from_bytes(
bytes: &[u8],
cipher: Option<crate::crypto::DatabaseCipher>,
) -> Result<Self> {
if bytes.len() < DATABASE_HEADER_SIZE {
return Err(Error::Corrupted(
"Byte buffer smaller than database header".into(),
));
}
let header = DatabaseHeader::from_bytes(&bytes[0..DATABASE_HEADER_SIZE])?;
let page_size = header.page_size as usize;
if page_size == 0 || !bytes.len().is_multiple_of(page_size) {
return Err(Error::Corrupted(
"Byte buffer length is not a valid multiple of page size".into(),
));
}
if header.encryption_flags == 1 {
if let Some(ref c) = cipher {
if !c.verify_kcv(&header.kcv) {
return Err(Error::DecryptionFailed(1));
}
} else {
return Err(Error::EncryptedDatabase);
}
}
let total_pages = (bytes.len() / page_size) as u32;
let mut in_memory_pages = HashMap::new();
for pid in 1..=total_pages {
let start = (pid as usize - 1) * page_size;
let end = start + page_size;
let disk_page = &bytes[start..end];
let page_data = if let Some(ref c) = cipher {
c.decrypt_page(pid, disk_page)?
} else {
disk_page.to_vec()
};
in_memory_pages.insert(pid, page_data);
}
let wal = Wal::open_in_memory(header.page_size as u32);
let mut pager = Self {
header,
file: None,
_path: None,
_lock_file: None,
_lock_path: None,
in_memory_pages: HashMap::new(),
wal: Some(wal),
wal_overlay: HashMap::new(),
concurrent_wal_path: None,
read_only: false,
concurrent_access: false,
busy_timeout: std::time::Duration::ZERO,
synced_wal_fingerprint: (0, 0, 0, 0),
synced_base_len: 0,
write_generation: 0,
checkpoint_threshold: DEFAULT_CHECKPOINT_THRESHOLD,
cipher,
transaction_savepoint: None,
named_savepoints: Vec::new(),
cache_capacity: 256,
lru_head: None,
lru_tail: None,
lru_nodes: HashMap::new(),
remote_reader: None,
remote_writer: None,
};
for (pid, page) in in_memory_pages {
pager.in_memory_pages.insert(pid, page);
pager.touch_lru(pid);
}
Ok(pager)
}
pub fn open_remote(
reader: Arc<dyn RemoteRangeReader>,
_cache_capacity: usize,
cipher: Option<crate::crypto::DatabaseCipher>,
) -> Result<Self> {
let total_size = reader.total_size()?;
if total_size < DATABASE_HEADER_SIZE as u64 {
return Err(Error::Corrupted(
"Remote file too small for database header".into(),
));
}
let header_buf = reader.fetch_range(0, DATABASE_HEADER_SIZE)?;
let header = DatabaseHeader::from_bytes(&header_buf)?;
if header.encryption_flags == 1 {
if let Some(ref c) = cipher {
if !c.verify_kcv(&header.kcv) {
return Err(Error::DecryptionFailed(1));
}
} else {
return Err(Error::EncryptedDatabase);
}
}
let cache_capacity = if _cache_capacity == 0 {
256
} else {
_cache_capacity
};
Ok(Self {
header,
file: None,
_path: None,
_lock_file: None,
_lock_path: None,
in_memory_pages: HashMap::new(),
wal: None,
wal_overlay: HashMap::new(),
concurrent_wal_path: None,
read_only: true,
concurrent_access: false,
busy_timeout: std::time::Duration::ZERO,
synced_wal_fingerprint: (0, 0, 0, 0),
synced_base_len: 0,
write_generation: 0,
checkpoint_threshold: DEFAULT_CHECKPOINT_THRESHOLD,
cipher,
transaction_savepoint: None,
named_savepoints: Vec::new(),
cache_capacity,
lru_head: None,
lru_tail: None,
lru_nodes: HashMap::new(),
remote_reader: Some(reader),
remote_writer: None,
})
}
pub fn open_remote_writable(
adapter: Arc<dyn RemoteStorageAdapter>,
_cache_capacity: usize,
cipher: Option<crate::crypto::DatabaseCipher>,
) -> Result<Self> {
let total_size = adapter.total_size()?;
let mut in_memory_pages = HashMap::new();
let header = if total_size >= DATABASE_HEADER_SIZE as u64 {
let header_buf = adapter.fetch_range(0, DATABASE_HEADER_SIZE)?;
let h = DatabaseHeader::from_bytes(&header_buf)?;
if h.encryption_flags == 1 {
if let Some(ref c) = cipher {
if !c.verify_kcv(&h.kcv) {
return Err(Error::DecryptionFailed(1));
}
} else {
return Err(Error::EncryptedDatabase);
}
}
h
} else if total_size == 0 {
let mut h = DatabaseHeader::new(DEFAULT_PAGE_SIZE);
if let Some(ref c) = cipher {
h.encryption_flags = 1;
h.salt = *c.salt();
h.kcv = c.generate_kcv()?;
}
let mut page1 = vec![0u8; DEFAULT_PAGE_SIZE as usize];
page1[0..DATABASE_HEADER_SIZE].copy_from_slice(&h.to_bytes());
in_memory_pages.insert(1, page1);
h
} else {
return Err(Error::Corrupted(
"Remote file too small for database header".into(),
));
};
let cache_capacity = if _cache_capacity == 0 {
256
} else {
_cache_capacity
};
let wal = Wal::open_in_memory(header.page_size as u32);
let mut pager = Self {
header,
file: None,
_path: None,
_lock_file: None,
_lock_path: None,
in_memory_pages,
wal: Some(wal),
wal_overlay: HashMap::new(),
concurrent_wal_path: None,
read_only: false,
concurrent_access: false,
busy_timeout: std::time::Duration::ZERO,
synced_wal_fingerprint: (0, 0, 0, 0),
synced_base_len: 0,
write_generation: 0,
checkpoint_threshold: DEFAULT_CHECKPOINT_THRESHOLD,
cipher,
transaction_savepoint: None,
named_savepoints: Vec::new(),
cache_capacity,
lru_head: None,
lru_tail: None,
lru_nodes: HashMap::new(),
remote_reader: Some(Arc::new(remote::AdapterRangeReader(adapter.clone()))),
remote_writer: Some(adapter),
};
if pager.in_memory_pages.contains_key(&1) {
pager.touch_lru(1);
}
Ok(pager)
}
pub fn push_remote(&mut self) -> Result<()> {
let writer = self.remote_writer.clone().ok_or_else(|| {
Error::Corrupted("No remote writable storage adapter configured".into())
})?;
let page_size = self.page_size();
let total_pages = self.header.total_pages.max(1);
let mut full_bytes = vec![0u8; total_pages as usize * page_size];
let h_buf = self.header.to_bytes();
full_bytes[0..DATABASE_HEADER_SIZE].copy_from_slice(&h_buf);
let page_1 = self.read_page(1)?;
full_bytes[DATABASE_HEADER_SIZE..page_size]
.copy_from_slice(&page_1[DATABASE_HEADER_SIZE..page_size]);
for pid in 2..=total_pages {
let page_data = self.read_page(pid)?;
let offset = (pid as usize - 1) * page_size;
full_bytes[offset..offset + page_size].copy_from_slice(&page_data);
}
writer.put_object(&full_bytes)
}
pub fn header(&self) -> &DatabaseHeader {
&self.header
}
pub fn header_mut(&mut self) -> &mut DatabaseHeader {
&mut self.header
}
pub fn page_size(&self) -> usize {
self.header.page_size as usize
}
pub fn total_pages(&self) -> u32 {
self.header.total_pages
}
pub fn cache_capacity(&self) -> usize {
self.cache_capacity
}
pub(crate) fn set_auto_checkpoint(&mut self, enabled: bool) {
self.checkpoint_threshold = if enabled {
DEFAULT_CHECKPOINT_THRESHOLD
} else {
usize::MAX
};
}
pub fn in_memory_page_count(&self) -> usize {
self.in_memory_pages.len()
}
pub fn sync_header(&mut self) -> Result<()> {
self.header.header_crc32 = self.header.calculate_crc();
let mut page = self.read_page(1)?;
page[..DATABASE_HEADER_SIZE].copy_from_slice(&self.header.to_bytes());
self.write_page(1, &page)
}
pub fn set_vector_index_page(&mut self, page_id: u32) -> Result<()> {
self.header.vector_index_page = page_id;
self.sync_header()
}
pub fn vector_index_page(&self) -> u32 {
self.header.vector_index_page
}
pub fn in_memory_pages(&self) -> &HashMap<PageId, Vec<u8>> {
&self.in_memory_pages
}
pub fn in_memory_pages_mut(&mut self) -> &mut HashMap<PageId, Vec<u8>> {
&mut self.in_memory_pages
}
fn touch_lru(&mut self, page_id: PageId) {
if self.lru_tail == Some(page_id) {
return;
}
if let Some(node) = self.lru_nodes.get(&page_id).copied() {
if let Some(prev) = node.prev {
if let Some(p) = self.lru_nodes.get_mut(&prev) {
p.next = node.next;
}
} else {
self.lru_head = node.next;
}
if let Some(next) = node.next {
if let Some(n) = self.lru_nodes.get_mut(&next) {
n.prev = node.prev;
}
} else {
self.lru_tail = node.prev;
}
}
let old_tail = self.lru_tail;
if let Some(tail_id) = old_tail {
if let Some(t) = self.lru_nodes.get_mut(&tail_id) {
t.next = Some(page_id);
}
self.lru_nodes.insert(
page_id,
LruNode {
prev: Some(tail_id),
next: None,
},
);
self.lru_tail = Some(page_id);
} else {
self.lru_head = Some(page_id);
self.lru_tail = Some(page_id);
self.lru_nodes.insert(
page_id,
LruNode {
prev: None,
next: None,
},
);
}
}
fn remove_lru(&mut self, page_id: PageId) {
if let Some(node) = self.lru_nodes.remove(&page_id) {
if let Some(prev) = node.prev {
if let Some(p) = self.lru_nodes.get_mut(&prev) {
p.next = node.next;
}
} else {
self.lru_head = node.next;
}
if let Some(next) = node.next {
if let Some(n) = self.lru_nodes.get_mut(&next) {
n.prev = node.prev;
}
} else {
self.lru_tail = node.prev;
}
}
}
fn evict_if_needed(&mut self) {
self.evict_if_needed_except(None);
}
fn evict_if_needed_except(&mut self, protected: Option<PageId>) {
if self.file.is_none() && self.remote_reader.is_none() {
return;
}
let capacity = self.cache_capacity.max(4);
while self.in_memory_pages.len() > capacity {
let mut curr = self.lru_head;
let mut evict_target = None;
while let Some(pid) = curr {
let next = self.lru_nodes.get(&pid).and_then(|n| n.next);
if pid != 1 && Some(pid) != protected {
let is_active_tx = if let Some(sp) = &self.transaction_savepoint {
sp.undo_pages.contains_key(&pid) || sp.dirty_pages.contains(&pid)
} else {
false
};
if !is_active_tx {
evict_target = Some(pid);
break;
}
}
curr = next;
}
if let Some(pid) = evict_target {
self.remove_lru(pid);
self.in_memory_pages.remove(&pid);
} else {
break;
}
}
}
pub fn is_compressed(&self) -> bool {
self.header.compression_flags == 1
}
pub fn enable_compression(&mut self) {
self.header.compression_flags = 1;
self.header.header_crc32 = self.header.calculate_crc();
}
fn decompress_if_needed(&self, page_id: PageId, raw: Vec<u8>) -> Result<Vec<u8>> {
let page_size = self.page_size();
if page_id > 1 && self.is_compressed() {
let decomp = compression::decompress_page_frame(&raw, page_size)?;
if decomp.len() < page_size {
let mut padded = decomp;
padded.resize(page_size, 0);
Ok(padded)
} else {
Ok(decomp)
}
} else {
Ok(raw)
}
}
fn load_page_uncached(&mut self, page_id: PageId) -> Result<Arc<Vec<u8>>> {
let page_size = self.page_size();
if let Some(data) = self.wal_overlay.get(&page_id) {
return Ok(Arc::clone(data));
}
if let Some(wal) = &self.wal
&& let Some(raw) = wal.read_page(page_id)
{
let plaintext = if let Some(cipher) = &self.cipher {
cipher.decrypt_page(page_id, &raw)?
} else {
raw
};
let final_page = self.decompress_if_needed(page_id, plaintext)?;
return Ok(Arc::new(final_page));
}
if let Some(file) = &mut self.file {
let offset = (page_id as u64 - 1) * page_size as u64;
file.seek(SeekFrom::Start(offset))?;
let mut disk_buf = vec![0u8; page_size];
file.read_exact(&mut disk_buf)?;
let plaintext = if let Some(cipher) = &self.cipher {
cipher.decrypt_page(page_id, &disk_buf)?
} else {
disk_buf
};
let final_page = self.decompress_if_needed(page_id, plaintext)?;
return Ok(Arc::new(final_page));
}
if let Some(reader) = &self.remote_reader {
let offset = (page_id as u64 - 1) * page_size as u64;
let raw_bytes = reader.fetch_range(offset, page_size)?;
let plaintext = if let Some(cipher) = &self.cipher {
cipher.decrypt_page(page_id, &raw_bytes)?
} else {
raw_bytes
};
let final_page = self.decompress_if_needed(page_id, plaintext)?;
return Ok(Arc::new(final_page));
}
Err(Error::PageNotFound(page_id))
}
fn ensure_cached_page(&mut self, page_id: PageId) -> Result<()> {
self.ensure_io_ready()?;
if page_id == 0 || page_id > self.header.total_pages {
return Err(Error::PageNotFound(page_id));
}
if self.in_memory_pages.contains_key(&page_id) {
self.touch_lru(page_id);
return Ok(());
}
let page_arc = self.load_page_uncached(page_id)?;
let final_page = (*page_arc).clone();
self.in_memory_pages.insert(page_id, final_page);
self.touch_lru(page_id);
self.evict_if_needed_except(Some(page_id));
Ok(())
}
pub fn read_page(&mut self, page_id: PageId) -> Result<Vec<u8>> {
self.ensure_cached_page(page_id)?;
self.in_memory_pages
.get(&page_id)
.cloned()
.ok_or(Error::PageNotFound(page_id))
}
pub fn with_page<R, F: FnOnce(&[u8]) -> Result<R>>(
&mut self,
page_id: PageId,
f: F,
) -> Result<R> {
self.ensure_io_ready()?;
if page_id == 0 || page_id > self.header.total_pages {
return Err(Error::PageNotFound(page_id));
}
if self.in_memory_pages.contains_key(&page_id) {
self.touch_lru(page_id);
let buf = &self.in_memory_pages[&page_id];
return f(buf);
}
if let Some(overlay_buf) = self.wal_overlay.get(&page_id) {
let arc_buf = Arc::clone(overlay_buf);
return f(&arc_buf);
}
self.ensure_cached_page(page_id)?;
let buf = self
.in_memory_pages
.get(&page_id)
.ok_or(Error::PageNotFound(page_id))?;
f(buf)
}
fn ensure_writable(&self, op: &str) -> Result<()> {
if self.remote_reader.is_some() && self.file.is_none() && self.remote_writer.is_none() {
return Err(Error::Corrupted(format!(
"Cannot {op} on a read-only remote storage connection"
)));
}
if self.read_only {
return Err(Error::TransactionError(format!(
"Database is opened read-only: cannot {op}"
)));
}
Ok(())
}
pub fn is_read_only(&self) -> bool {
self.read_only
}
pub fn is_concurrent_access(&self) -> bool {
self.concurrent_access
}
pub fn set_busy_timeout(&mut self, timeout: std::time::Duration) {
self.busy_timeout = timeout;
}
pub fn busy_timeout(&self) -> std::time::Duration {
self.busy_timeout
}
pub fn write_generation(&self) -> u64 {
self.write_generation
}
pub fn modify_page<R, F: FnOnce(&mut [u8]) -> Result<R>>(
&mut self,
page_id: PageId,
f: F,
) -> Result<R> {
self.ensure_writable("modify page")?;
self.ensure_cached_page(page_id)?;
let needs_undo = self
.transaction_savepoint
.as_ref()
.map(|sp| (!sp.undo_pages.contains_key(&page_id), sp.total_pages));
if let Some((true, total_pages)) = needs_undo {
let prev_data = if let Some(cached) = self.in_memory_pages.get(&page_id) {
Some(cached.clone())
} else if page_id <= total_pages {
Some(self.read_page(page_id)?)
} else {
None
};
if let Some(sp) = &mut self.transaction_savepoint {
sp.undo_pages.insert(page_id, prev_data.clone());
}
if let Some(sp) = self.named_savepoints.last_mut() {
sp.undo_pages.entry(page_id).or_insert(prev_data);
}
} else if let Some(sp) = self.named_savepoints.last_mut()
&& let std::collections::hash_map::Entry::Vacant(e) = sp.undo_pages.entry(page_id)
{
let prev_data = self.in_memory_pages.get(&page_id).cloned();
e.insert(prev_data);
}
let is_in_tx = self.is_in_transaction();
if is_in_tx {
if let Some(sp) = &mut self.transaction_savepoint {
sp.dirty_pages.insert(page_id);
}
if let Some(sp) = self.named_savepoints.last_mut() {
sp.dirty_pages.insert(page_id);
}
}
let buf = self
.in_memory_pages
.get_mut(&page_id)
.ok_or(Error::PageNotFound(page_id))?;
let res = f(buf)?;
if page_id == 1 {
let header_bytes = self.header.to_bytes();
let len = header_bytes.len().min(buf.len());
buf[..len].copy_from_slice(&header_bytes[..len]);
}
if !is_in_tx {
let clone_for_write = buf.clone();
self.write_page(page_id, &clone_for_write)?;
}
Ok(res)
}
pub fn write_page(&mut self, page_id: PageId, data: &[u8]) -> Result<()> {
self.ensure_writable("write page")?;
if self.file.is_none() && self.remote_writer.is_some() {
self.in_memory_pages.insert(page_id, data.to_vec());
self.touch_lru(page_id);
return Ok(());
}
let page_size = self.page_size();
if data.len() != page_size {
return Err(Error::Corrupted(format!(
"Invalid page data length: expected {}, got {}",
page_size,
data.len()
)));
}
if page_id == 0 || page_id > self.header.total_pages {
return Err(Error::PageNotFound(page_id));
}
let patched_p1: Vec<u8>;
let data: &[u8] = if page_id == 1 {
let mut p = data.to_vec();
let header_bytes = self.header.to_bytes();
let len = header_bytes.len().min(p.len());
p[..len].copy_from_slice(&header_bytes[..len]);
patched_p1 = p;
&patched_p1
} else {
data
};
let needs_undo = self
.transaction_savepoint
.as_ref()
.map(|sp| (!sp.undo_pages.contains_key(&page_id), sp.total_pages));
if let Some((true, total_pages)) = needs_undo {
let prev_data = if let Some(cached) = self.in_memory_pages.get(&page_id) {
Some(cached.clone())
} else if page_id <= total_pages {
Some(self.read_page(page_id)?)
} else {
None
};
if let Some(sp) = &mut self.transaction_savepoint {
sp.undo_pages.insert(page_id, prev_data.clone());
}
if let Some(sp) = self.named_savepoints.last_mut() {
sp.undo_pages.entry(page_id).or_insert(prev_data);
}
} else if let Some(sp) = self.named_savepoints.last_mut()
&& let std::collections::hash_map::Entry::Vacant(e) = sp.undo_pages.entry(page_id)
{
let prev_data = self.in_memory_pages.get(&page_id).cloned();
e.insert(prev_data);
}
self.in_memory_pages.insert(page_id, data.to_vec());
let is_in_tx = self.is_in_transaction();
if is_in_tx {
if let Some(sp) = &mut self.transaction_savepoint {
sp.dirty_pages.insert(page_id);
}
if let Some(sp) = self.named_savepoints.last_mut() {
sp.dirty_pages.insert(page_id);
}
return Ok(());
}
self.touch_lru(page_id);
self.evict_if_needed();
let payload = if self.is_compressed() && page_id > 1 {
let mut comp = compression::compress_page_frame(data);
if comp.len() < page_size {
comp.resize(page_size, 0);
}
comp
} else {
data.to_vec()
};
let disk_bytes = if let Some(cipher) = &self.cipher {
cipher.encrypt_page(page_id, &payload)?
} else {
payload
};
let should_checkpoint = if let Some(wal) = &mut self.wal {
wal.write_frame(page_id, &disk_bytes, true, self.header.total_pages)?;
wal.frame_count() >= self.checkpoint_threshold
} else {
if let Some(file) = &mut self.file {
let offset = (page_id as u64 - 1) * page_size as u64;
file.seek(SeekFrom::Start(offset))?;
file.write_all(&disk_bytes)?;
file.sync_data()?;
}
false
};
if should_checkpoint {
self.checkpoint()?;
}
self.write_generation = self.write_generation.wrapping_add(1);
Ok(())
}
pub fn allocate_page(&mut self) -> Result<PageId> {
self.ensure_writable("allocate page")?;
let page_size = self.page_size();
let mut reused_from_freelist = false;
let allocated_id = if self.header.freelist_count > 0 && self.header.freelist_trunk > 0 {
let trunk_id = self.header.freelist_trunk;
let mut trunk_page = self.read_page(trunk_id)?;
let count =
u32::from_le_bytes([trunk_page[4], trunk_page[5], trunk_page[6], trunk_page[7]])
as usize;
if count > 0 {
if count > page_size.saturating_sub(8) / 4 {
return Err(Error::Corrupted(
"Freelist slot count exceeds page capacity".into(),
));
}
let offset = 8 + (count - 1) * 4;
let reused_id = u32::from_le_bytes([
trunk_page[offset],
trunk_page[offset + 1],
trunk_page[offset + 2],
trunk_page[offset + 3],
]);
let new_count = (count - 1) as u32;
trunk_page[4..8].copy_from_slice(&new_count.to_le_bytes());
self.write_page(trunk_id, &trunk_page)?;
self.header.freelist_count = self.header.freelist_count.saturating_sub(1);
reused_from_freelist = true;
reused_id
} else {
let next_trunk = u32::from_le_bytes([
trunk_page[0],
trunk_page[1],
trunk_page[2],
trunk_page[3],
]);
self.header.freelist_trunk = next_trunk;
self.header.freelist_count = self.header.freelist_count.saturating_sub(1);
reused_from_freelist = true;
trunk_id
}
} else {
let new_id = self.header.total_pages + 1;
self.header.total_pages = new_id;
new_id
};
let allocation_original = if reused_from_freelist {
Some(self.read_page(allocated_id)?)
} else {
None
};
if let Some(sp) = &mut self.transaction_savepoint {
sp.undo_pages
.entry(allocated_id)
.or_insert_with(|| allocation_original.clone());
}
if let Some(sp) = self.named_savepoints.last_mut() {
sp.undo_pages
.entry(allocated_id)
.or_insert(allocation_original);
}
let empty_page = vec![0u8; page_size];
let is_in_tx = self.is_in_transaction();
if !reused_from_freelist {
if self.file.is_none() && self.remote_writer.is_some() {
self.in_memory_pages.insert(allocated_id, empty_page);
self.sync_header_into_page1_cache();
return Ok(allocated_id);
}
if !is_in_tx
&& self.wal.is_none()
&& let Some(file) = &mut self.file
{
let offset = (allocated_id as u64 - 1) * page_size as u64;
file.seek(SeekFrom::Start(offset))?;
file.write_all(&empty_page)?;
file.sync_data()?;
}
}
self.in_memory_pages.insert(allocated_id, empty_page);
self.touch_lru(allocated_id);
self.evict_if_needed();
self.sync_header_into_page1_cache();
if let Some(page1_data) = self.in_memory_pages.get(&1).cloned() {
let is_in_tx = self.transaction_savepoint.is_some();
if is_in_tx {
if let Some(sp) = &mut self.transaction_savepoint {
sp.dirty_pages.insert(1);
sp.dirty_pages.insert(allocated_id);
}
if let Some(sp) = self.named_savepoints.last_mut() {
sp.dirty_pages.insert(1);
sp.dirty_pages.insert(allocated_id);
}
} else {
let payload = if let Some(cipher) = &self.cipher {
cipher.encrypt_page(1, &page1_data)?
} else {
page1_data.clone()
};
if let Some(wal) = &mut self.wal {
wal.write_frame(1, &payload, true, self.header.total_pages)?;
} else if let Some(file) = &mut self.file {
let header_bytes = self.header.to_bytes();
file.seek(SeekFrom::Start(0))?;
file.write_all(&header_bytes)?;
file.sync_data()?;
}
}
}
Ok(allocated_id)
}
pub fn free_page(&mut self, page_id: PageId) -> Result<()> {
if page_id == 0 || page_id > self.header.total_pages || page_id == 1 {
return Err(Error::Corrupted(format!("Invalid page {page_id} to free")));
}
let page_size = self.page_size();
let max_leaf_slots = page_size.saturating_sub(8) / 4;
if self.header.freelist_trunk == 0 {
let mut trunk_page = vec![0u8; page_size];
trunk_page[0..4].copy_from_slice(&0u32.to_le_bytes()); trunk_page[4..8].copy_from_slice(&0u32.to_le_bytes()); self.write_page(page_id, &trunk_page)?;
self.header.freelist_trunk = page_id;
self.header.freelist_count = 1;
} else {
let trunk_id = self.header.freelist_trunk;
let mut trunk_page = self.read_page(trunk_id)?;
let count =
u32::from_le_bytes([trunk_page[4], trunk_page[5], trunk_page[6], trunk_page[7]])
as usize;
if count < max_leaf_slots {
let offset = 8 + count * 4;
trunk_page[offset..offset + 4].copy_from_slice(&page_id.to_le_bytes());
let new_count = (count + 1) as u32;
trunk_page[4..8].copy_from_slice(&new_count.to_le_bytes());
self.write_page(trunk_id, &trunk_page)?;
self.header.freelist_count += 1;
} else {
let mut new_trunk = vec![0u8; page_size];
new_trunk[0..4].copy_from_slice(&trunk_id.to_le_bytes());
new_trunk[4..8].copy_from_slice(&0u32.to_le_bytes());
self.write_page(page_id, &new_trunk)?;
self.header.freelist_trunk = page_id;
self.header.freelist_count += 1;
}
}
self.sync_header_into_page1_cache();
Ok(())
}
fn sync_header_into_page1_cache(&mut self) {
let header_bytes = self.header.to_bytes();
let page_size = self.header.page_size as usize;
let page1 = self
.in_memory_pages
.entry(1)
.or_insert_with(|| vec![0u8; page_size]);
let len = header_bytes.len().min(page1.len());
page1[..len].copy_from_slice(&header_bytes[..len]);
}
pub fn checkpoint(&mut self) -> Result<usize> {
if let Some(mut wal) = self.wal.take() {
let result = if self.file.is_none() && (self.is_compressed() || self.cipher.is_some()) {
let mut discard_sink = HashMap::new();
wal.checkpoint(&mut None, &mut discard_sink)
} else {
wal.checkpoint(&mut self.file, &mut self.in_memory_pages)
};
self.wal = Some(wal);
result
} else {
Ok(0)
}
}
pub fn storage_health(&self) -> StorageHealth {
self.wal.as_ref().map(Wal::health).unwrap_or(StorageHealth {
reopen_required: false,
checkpoint_error: None,
wal_frames: 0,
})
}
#[cfg(feature = "fault-injection")]
pub fn set_io_fault(&self, plan: Option<fault::IoFaultPlan>) -> Result<()> {
self.wal
.as_ref()
.ok_or_else(|| Error::TransactionError("No WAL is available".into()))?
.set_io_fault(plan)
}
pub fn sync(&mut self) -> Result<()> {
self.checkpoint()?;
if let Some(file) = &mut self.file {
file.sync_all()?;
}
Ok(())
}
pub fn is_in_transaction(&self) -> bool {
self.transaction_savepoint.is_some()
}
pub fn begin_transaction(&mut self) -> Result<()> {
self.ensure_io_ready()?;
self.ensure_writable("begin transaction")?;
if self.transaction_savepoint.is_some() {
return Err(Error::TransactionError(
"Transaction already in progress".into(),
));
}
let frame_count = self.wal.as_ref().map(|w| w.frame_count()).unwrap_or(0);
self.transaction_savepoint = Some(TransactionSavepoint {
header: self.header.clone(),
undo_pages: HashMap::new(),
dirty_pages: HashSet::new(),
total_pages: self.header.total_pages,
wal_frame_count: frame_count,
});
Ok(())
}
pub fn commit_transaction(&mut self) -> Result<()> {
let savepoint = self
.transaction_savepoint
.as_ref()
.ok_or_else(|| Error::TransactionError("No active transaction to commit".into()))?;
let start_frames = savepoint.wal_frame_count;
self.header.change_counter = savepoint
.header
.change_counter
.checked_add(1)
.ok_or_else(|| Error::TransactionError("Commit sequence exhausted".into()))?;
self.sync_header_into_page1_cache();
self.transaction_savepoint
.as_mut()
.unwrap()
.dirty_pages
.insert(1);
let mut dirty: Vec<_> = self
.transaction_savepoint
.as_ref()
.unwrap()
.dirty_pages
.iter()
.copied()
.filter(|pid| *pid <= self.header.total_pages && self.in_memory_pages.contains_key(pid))
.collect();
dirty.sort_unstable();
let page_size = self.page_size();
let result = (|| -> Result<()> {
for (position, pid) in dirty.iter().copied().enumerate() {
let page = &self.in_memory_pages[&pid];
let mut payload = if self.is_compressed() && pid > 1 {
compression::compress_page_frame(page)
} else {
page.clone()
};
payload.resize(page_size, 0);
let bytes = if let Some(cipher) = &self.cipher {
cipher.encrypt_page(pid, &payload)?
} else {
payload
};
let last = position + 1 == dirty.len();
if let Some(wal) = &mut self.wal {
wal.write_frame(pid, &bytes, last, self.header.total_pages)?;
} else if let Some(file) = &mut self.file {
file.seek(SeekFrom::Start((pid as u64 - 1) * page_size as u64))?;
file.write_all(&bytes)?;
if last {
file.sync_data()?;
}
}
}
Ok(())
})();
if let Err(error) = result {
if let Some(wal) = &mut self.wal {
wal.rollback_to(start_frames)?;
}
return Err(error);
}
self.transaction_savepoint = None;
self.write_generation = self.write_generation.wrapping_add(1);
self.named_savepoints.clear();
if self
.wal
.as_ref()
.is_some_and(|wal| wal.frame_count() >= self.checkpoint_threshold)
{
let _ = self.checkpoint();
}
self.evict_if_needed();
Ok(())
}
pub fn rollback_transaction(&mut self) -> Result<()> {
let savepoint = self
.transaction_savepoint
.take()
.ok_or_else(|| Error::TransactionError("No active transaction to rollback".into()))?;
self.named_savepoints.clear();
for (page_id, original_data) in savepoint.undo_pages {
match original_data {
Some(prev) => {
self.in_memory_pages.insert(page_id, prev);
self.touch_lru(page_id);
}
None => {
self.in_memory_pages.remove(&page_id);
self.remove_lru(page_id);
}
}
}
self.header = savepoint.header;
self.sync_header_into_page1_cache();
if let Some(wal) = &mut self.wal {
wal.rollback_to(savepoint.wal_frame_count)?;
}
Ok(())
}
pub fn create_savepoint(&mut self, name: &str) -> Result<()> {
if self.transaction_savepoint.is_none() {
self.begin_transaction()?;
}
let frame_count = self.wal.as_ref().map(|w| w.frame_count()).unwrap_or(0);
self.named_savepoints.push(NamedSavepoint {
header: self.header.clone(),
name: name.to_string(),
undo_pages: HashMap::new(),
dirty_pages: HashSet::new(),
total_pages: self.header.total_pages,
wal_frame_count: frame_count,
});
Ok(())
}
pub(crate) fn savepoint_depth(&self) -> usize {
self.named_savepoints.len()
}
pub(crate) fn savepoint_position(&self, name: &str) -> Option<usize> {
self.named_savepoints
.iter()
.rposition(|sp| sp.name.eq_ignore_ascii_case(name))
}
pub fn rollback_to_savepoint(&mut self, name: &str) -> Result<()> {
let idx = self
.named_savepoints
.iter()
.rposition(|sp| sp.name.eq_ignore_ascii_case(name))
.ok_or_else(|| Error::TransactionError(format!("Savepoint '{name}' does not exist")))?;
while self.named_savepoints.len() > idx {
let sp = self.named_savepoints.pop().unwrap();
for (page_id, original_data) in sp.undo_pages {
match original_data {
Some(prev) => {
self.in_memory_pages.insert(page_id, prev);
self.touch_lru(page_id);
}
None => {
self.in_memory_pages.remove(&page_id);
self.remove_lru(page_id);
}
}
}
self.header = sp.header;
self.sync_header_into_page1_cache();
if let Some(wal) = &mut self.wal {
wal.rollback_to(sp.wal_frame_count)?;
}
}
self.create_savepoint(name)?;
Ok(())
}
pub fn release_savepoint(&mut self, name: &str) -> Result<()> {
let idx = self
.named_savepoints
.iter()
.rposition(|sp| sp.name.eq_ignore_ascii_case(name))
.ok_or_else(|| Error::TransactionError(format!("Savepoint '{name}' does not exist")))?;
let released = self.named_savepoints.split_off(idx);
if let Some(parent) = self.named_savepoints.last_mut() {
for child in released {
for (pid, before) in child.undo_pages {
parent.undo_pages.entry(pid).or_insert(before);
}
parent.dirty_pages.extend(child.dirty_pages);
}
}
Ok(())
}
pub(crate) fn require_committed_snapshot(&self) -> Result<()> {
self.ensure_io_ready()?;
if self.is_in_transaction() {
return Err(Error::TransactionError(
"Commit or roll back the active transaction before exporting a snapshot".into(),
));
}
Ok(())
}
pub(crate) fn committed_pages(&self) -> Result<Vec<(PageId, Vec<u8>)>> {
self.require_committed_snapshot()?;
let mut pages = Vec::with_capacity(self.header.total_pages as usize);
let mut file = self.file.as_ref().map(File::try_clone).transpose()?;
for pid in 1..=self.header.total_pages {
if let Some(page) = self.in_memory_pages.get(&pid) {
pages.push((pid, page.clone()));
continue;
}
let raw = if let Some(raw) = self.wal.as_ref().and_then(|wal| wal.read_page(pid)) {
raw
} else if let Some(file) = &mut file {
let mut raw = vec![0; self.page_size()];
file.seek(SeekFrom::Start((pid as u64 - 1) * self.page_size() as u64))?;
file.read_exact(&mut raw)?;
raw
} else if let Some(reader) = &self.remote_reader {
reader.fetch_range((pid as u64 - 1) * self.page_size() as u64, self.page_size())?
} else {
return Err(Error::PageNotFound(pid));
};
let plain = if let Some(cipher) = &self.cipher {
cipher.decrypt_page(pid, &raw)?
} else {
raw
};
pages.push((pid, self.decompress_if_needed(pid, plain)?));
}
Ok(pages)
}
#[cfg(not(target_arch = "wasm32"))]
pub fn backup_to(&mut self, dest_path: &Path) -> Result<u32> {
self.backup_to_options(dest_path, true)
}
#[cfg(not(target_arch = "wasm32"))]
pub(crate) fn backup_to_options(&mut self, dest_path: &Path, overwrite: bool) -> Result<u32> {
self.require_committed_snapshot()?;
if let Some(active_db) = &self._path {
if let (Ok(canon_dest), Ok(canon_active)) =
(dest_path.canonicalize(), active_db.canonicalize())
{
if canon_dest == canon_active {
return Err(Error::SqlSyntax(
"Cannot VACUUM INTO the currently active database file".into(),
));
}
} else if dest_path == active_db.as_path() {
return Err(Error::SqlSyntax(
"Cannot VACUUM INTO the currently active database file".into(),
));
}
}
struct DestinationLock {
file: Option<File>,
path: PathBuf,
}
impl Drop for DestinationLock {
fn drop(&mut self) {
self.file.take();
let _ = std::fs::remove_file(&self.path);
}
}
let lock_path = dest_path.with_extension("tapir-lock");
let lock_file = OpenOptions::new()
.write(true)
.create_new(true)
.open(&lock_path)
.map_err(|error| {
if error.kind() == std::io::ErrorKind::AlreadyExists {
Error::Busy("Backup/restore destination is locked".into())
} else {
error.into()
}
})?;
let mut destination_lock = DestinationLock {
file: Some(lock_file),
path: lock_path,
};
write!(
destination_lock.file.as_mut().expect("destination lock"),
"{}",
std::process::id()
)?;
destination_lock
.file
.as_mut()
.expect("destination lock")
.flush()?;
if !overwrite && dest_path.exists() {
return Err(Error::ConstraintViolation(
"Restore destination already exists".into(),
));
}
let old_wal = dest_path.with_extension("tapir-wal");
if let Ok(metadata) = old_wal.metadata() {
if metadata.len() > wal::WAL_HEADER_SIZE as u64 {
return Err(Error::Busy(
"Checkpoint and close the destination WAL before replacing its database".into(),
));
}
std::fs::remove_file(&old_wal)?;
}
let parent = dest_path
.parent()
.filter(|path| !path.as_os_str().is_empty())
.unwrap_or_else(|| Path::new("."));
let mut staging = tempfile::NamedTempFile::new_in(parent)?;
let io = self.wal.as_ref().map(Wal::storage_io).unwrap_or_default();
let total = self.header.total_pages;
let page_size = self.page_size();
for pid in 1..=total {
let page_data = self.read_page(pid)?;
let disk_page = if let Some(ref c) = self.cipher {
c.encrypt_page(pid, &page_data)?
} else {
page_data
};
staging
.as_file_mut()
.seek(SeekFrom::Start((pid as u64 - 1) * page_size as u64))?;
io.write(
fault::IoPoint::BackupWrite,
staging.as_file_mut(),
&disk_page,
)?;
}
io.before(fault::IoPoint::BackupSync)?;
staging.as_file().sync_all()?;
let _installed = (if overwrite {
staging.persist(dest_path)
} else {
staging.persist_noclobber(dest_path)
})
.map_err(|error| Error::Io(error.error))?;
#[cfg(unix)]
File::open(parent)?.sync_all()?;
Ok(total)
}
#[cfg(target_arch = "wasm32")]
pub fn backup_to(&mut self, _dest_path: &Path) -> Result<u32> {
Err(std::io::Error::new(
std::io::ErrorKind::Unsupported,
"Use export_bytes in browser WASM",
)
.into())
}
pub fn export_bytes(&mut self) -> Result<Vec<u8>> {
self.require_committed_snapshot()?;
let _ = self.checkpoint()?;
let total = self.header.total_pages;
let page_size = self.page_size();
let mut buffer = Vec::with_capacity(total as usize * page_size);
for pid in 1..=total {
let page_data = self.read_page(pid)?;
let disk_page = if let Some(ref c) = self.cipher {
c.encrypt_page(pid, &page_data)?
} else {
page_data
};
buffer.extend_from_slice(&disk_page);
}
Ok(buffer)
}
pub fn export_wal_frames(&self) -> Result<Vec<(PageId, Vec<u8>)>> {
if let Some(wal) = &self.wal {
Ok(wal.export_frames())
} else {
Ok(Vec::new())
}
}
pub fn apply_wal_frame(&mut self, page_id: PageId, data: &[u8]) -> Result<()> {
self.write_page(page_id, data)
}
pub fn refresh_committed_state(&mut self) -> Result<bool> {
let Some(db_path) = self._path.clone() else {
return Ok(false);
};
if !self.read_only && !self.concurrent_access {
return Ok(false);
}
if self.is_in_transaction() || self.wal.is_some() {
return Ok(false);
}
let wal_path = self
.concurrent_wal_path
.clone()
.unwrap_or_else(|| db_path.with_extension("tapir-wal"));
for attempt in 0..10 {
let wal_fp = Wal::read_file_fingerprint(&wal_path).unwrap_or((0, 0, 0, 0));
let base_len = self
.file
.as_ref()
.and_then(|f| f.metadata().ok())
.map(|m| m.len())
.unwrap_or(0);
if attempt == 0
&& wal_fp == self.synced_wal_fingerprint
&& base_len == self.synced_base_len
{
return Ok(false);
}
let wal_state = match Wal::read_all_committed_pages(&wal_path) {
Ok(s) => s,
Err(_) if attempt < 9 => {
std::thread::sleep(std::time::Duration::from_millis(3));
continue;
}
Err(e) => return Err(e),
};
let mut overlay = HashMap::new();
let mut wal_db_size = None;
let mut retry_needed = false;
if let Some((db_size, raw_pages)) = wal_state {
wal_db_size = Some(db_size);
for (pid, raw) in raw_pages {
let pt = if let Some(ref c) = self.cipher {
match c.decrypt_page(pid, &raw) {
Ok(p) => p,
Err(_) if attempt < 9 => {
retry_needed = true;
break;
}
Err(e) => return Err(e),
}
} else {
raw
};
let decomp = self.decompress_if_needed(pid, pt)?;
overlay.insert(pid, decomp);
}
}
if retry_needed {
std::thread::sleep(std::time::Duration::from_millis(3));
continue;
}
let new_header = if let Some(p1) = overlay.get(&1) {
DatabaseHeader::from_bytes(&p1[..DATABASE_HEADER_SIZE])?
} else if let Some(ref mut f) = self.file {
f.seek(SeekFrom::Start(0))?;
let mut h_buf = [0u8; DATABASE_HEADER_SIZE];
f.read_exact(&mut h_buf)?;
DatabaseHeader::from_bytes(&h_buf)?
} else {
self.header.clone()
};
self.header = new_header;
if let Some(sz) = wal_db_size
&& sz > self.header.total_pages
{
self.header.total_pages = sz;
}
self.wal_overlay = overlay.into_iter().map(|(k, v)| (k, Arc::new(v))).collect();
self.in_memory_pages.clear();
self.lru_nodes.clear();
self.lru_head = None;
self.lru_tail = None;
self.synced_wal_fingerprint = wal_fp;
self.synced_base_len = base_len;
return Ok(true);
}
Ok(false)
}
pub fn begin_concurrent_write(&mut self) -> Result<bool> {
if !self.concurrent_access {
return Ok(false);
}
if self.read_only {
return Err(Error::TransactionError(
"Database is opened read-only: cannot acquire write lock".into(),
));
}
if self._lock_file.is_some() {
return Ok(false);
}
let db_path = self
._path
.clone()
.ok_or_else(|| Error::TransactionError("Concurrent mode requires file path".into()))?;
let lock_path = self
._lock_path
.clone()
.unwrap_or_else(|| db_path.with_extension("tapir-lock"));
let wal_path = self
.concurrent_wal_path
.clone()
.unwrap_or_else(|| db_path.with_extension("tapir-wal"));
let lock_file = Self::acquire_lock_file(&db_path, &lock_path, self.busy_timeout)?;
let refreshed = match self.refresh_committed_state() {
Ok(changed) => changed,
Err(e) => {
drop(lock_file);
let _ = std::fs::remove_file(&lock_path);
return Err(e);
}
};
let wal = match Wal::open(&wal_path, self.header.page_size as u32) {
Ok(w) => w,
Err(e) => {
drop(lock_file);
let _ = std::fs::remove_file(&lock_path);
return Err(e);
}
};
self._lock_file = Some(lock_file);
self.wal = Some(wal);
Ok(refreshed)
}
pub fn end_concurrent_write(&mut self) -> Result<()> {
if !self.concurrent_access || self.is_in_transaction() || self._lock_file.is_none() {
return Ok(());
}
if let Some(wal) = self.wal.take() {
if wal.frame_count() == 0 {
self.wal_overlay.clear();
} else {
for (pid, _) in wal.export_frames() {
if let Some(pt) = self.in_memory_pages.get(&pid) {
self.wal_overlay.insert(pid, Arc::new(pt.clone()));
}
}
}
}
if let Some(ref db_path) = self._path {
let wal_path = self
.concurrent_wal_path
.clone()
.unwrap_or_else(|| db_path.with_extension("tapir-wal"));
self.synced_wal_fingerprint =
Wal::read_file_fingerprint(&wal_path).unwrap_or((0, 0, 0, 0));
self.synced_base_len = self
.file
.as_ref()
.and_then(|f| f.metadata().ok())
.map(|m| m.len())
.unwrap_or(0);
}
if self._lock_file.take().is_some()
&& let Some(ref lock_path) = self._lock_path
{
let _ = std::fs::remove_file(lock_path);
}
Ok(())
}
pub fn snapshot_clone(&mut self) -> Result<Self> {
self.ensure_io_ready()?;
let pages = if self.file.is_none()
&& self.wal.is_none()
&& self.in_memory_pages.is_empty()
&& !self.wal_overlay.is_empty()
{
self.wal_overlay.clone()
} else {
let total = self.header.total_pages;
let mut map = HashMap::with_capacity(total as usize);
for pid in 1..=total {
if let Some(cached) = self.in_memory_pages.get(&pid) {
map.insert(pid, Arc::new(cached.clone()));
} else {
map.insert(pid, self.load_page_uncached(pid)?);
}
}
map
};
Ok(Self {
header: self.header.clone(),
file: None,
_path: None,
_lock_file: None,
_lock_path: None,
in_memory_pages: HashMap::new(),
wal: None,
wal_overlay: pages,
concurrent_wal_path: None,
read_only: true,
concurrent_access: false,
busy_timeout: std::time::Duration::ZERO,
synced_wal_fingerprint: (0, 0, 0, 0),
synced_base_len: 0,
write_generation: 0,
checkpoint_threshold: self.checkpoint_threshold,
cipher: None,
transaction_savepoint: None,
named_savepoints: Vec::new(),
cache_capacity: self.cache_capacity.max(256),
lru_head: None,
lru_tail: None,
lru_nodes: HashMap::new(),
remote_reader: None,
remote_writer: None,
})
}
}
impl Drop for Pager {
fn drop(&mut self) {
if self._lock_file.take().is_some()
&& let Some(ref lock_path) = self._lock_path
{
let _ = std::fs::remove_file(lock_path);
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_header_serialization_roundtrip() {
let header = DatabaseHeader::default();
let bytes = header.to_bytes();
let parsed = DatabaseHeader::from_bytes(&bytes).expect("Header parse failed");
assert_eq!(header, parsed);
}
#[test]
fn test_in_memory_pager_read_write_alloc() {
let mut pager = Pager::open_in_memory(4096, 128).expect("Failed to open in-memory pager");
assert_eq!(pager.total_pages(), 1);
let page1 = pager.read_page(1).expect("Failed to read page 1");
assert_eq!(page1.len(), 4096);
assert_eq!(&page1[0..8], b"TAPIRUS\0");
let page2_id = pager.allocate_page().expect("Failed to allocate page 2");
assert_eq!(page2_id, 2);
assert_eq!(pager.total_pages(), 2);
let mut custom_data = vec![42u8; 4096];
custom_data[0..4].copy_from_slice(b"TEST");
pager
.write_page(page2_id, &custom_data)
.expect("Failed write");
let read_back = pager.read_page(page2_id).expect("Failed to read back");
assert_eq!(&read_back[0..4], b"TEST");
assert_eq!(read_back[10], 42);
}
#[test]
fn test_transaction_undo_rollback_and_page_restore() {
let mut pager = Pager::open_in_memory(4096, 128).expect("Open in-memory");
let page2_id = pager.allocate_page().expect("Allocate 2");
let data1 = vec![1u8; 4096];
pager.write_page(page2_id, &data1).expect("Write page 2");
pager.begin_transaction().expect("Begin transaction");
assert!(pager.is_in_transaction());
let data2 = vec![2u8; 4096];
pager.write_page(page2_id, &data2).expect("Modify page 2");
assert_eq!(pager.read_page(page2_id).unwrap()[0], 2);
let _page3_id = pager.allocate_page().expect("Allocate 3");
assert_eq!(pager.total_pages(), 3);
pager.rollback_transaction().expect("Rollback transaction");
assert!(!pager.is_in_transaction());
assert_eq!(pager.read_page(page2_id).unwrap()[0], 1);
assert_eq!(pager.total_pages(), 2);
}
#[test]
fn test_pager_lru_eviction_on_disk() {
let temp_dir = std::env::temp_dir();
let db_path = temp_dir.join(format!(
"tapirus_lru_test_{}.tapir",
web_time::SystemTime::now()
.duration_since(web_time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
{
let mut pager =
Pager::open_with_config(&db_path, 4096, 5).expect("Open with capacity 5");
assert_eq!(pager.cache_capacity(), 5);
for i in 2..=20 {
let pid = pager.allocate_page().expect("Allocate");
let mut pdata = vec![i as u8; 4096];
pdata[0] = i as u8;
pager.write_page(pid, &pdata).expect("Write");
}
pager.sync().expect("Sync");
assert!(pager.in_memory_page_count() <= 6);
let p2 = pager.read_page(2).expect("Reload evicted page 2");
assert_eq!(p2[0], 2);
}
let _ = std::fs::remove_file(&db_path);
let _ = std::fs::remove_file(db_path.with_extension("tapir-wal"));
}
#[test]
fn test_freelist_page_allocation_and_reuse() {
let mut pager = Pager::open_in_memory(4096, 128).expect("Open in-memory pager");
assert_eq!(pager.total_pages(), 1);
let p2 = pager.allocate_page().unwrap();
let p3 = pager.allocate_page().unwrap();
let p4 = pager.allocate_page().unwrap();
assert_eq!(p2, 2);
assert_eq!(p3, 3);
assert_eq!(p4, 4);
assert_eq!(pager.total_pages(), 4);
assert_eq!(pager.header().freelist_count, 0);
pager.free_page(p3).expect("Free page 3");
assert_eq!(pager.header().freelist_count, 1);
assert_eq!(pager.header().freelist_trunk, 3);
let reused_page = pager.allocate_page().expect("Allocate reused page");
assert_eq!(reused_page, 3);
assert_eq!(pager.total_pages(), 4);
assert_eq!(pager.header().freelist_count, 0);
let p5 = pager.allocate_page().expect("Allocate page 5");
assert_eq!(p5, 5);
assert_eq!(pager.total_pages(), 5);
}
#[test]
fn test_lru_order_preservation_and_capacity() {
let temp_dir = std::env::temp_dir();
let db_path = temp_dir.join(format!(
"tapirus_lru_order_test_{}.tapir",
web_time::SystemTime::now()
.duration_since(web_time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
{
let mut pager =
Pager::open_with_config(&db_path, 4096, 4).expect("Open with capacity 4");
for pid in 2..=4 {
let p = pager.allocate_page().unwrap();
assert_eq!(p, pid);
let pdata = vec![pid as u8; 4096];
pager.write_page(p, &pdata).unwrap();
}
pager.sync().unwrap();
assert!(pager.in_memory_pages().contains_key(&2));
assert!(pager.in_memory_pages().contains_key(&3));
assert!(pager.in_memory_pages().contains_key(&4));
let _ = pager.read_page(2).unwrap();
let p5 = pager.allocate_page().unwrap();
let pdata5 = vec![5u8; 4096];
pager.write_page(p5, &pdata5).unwrap();
assert!(
pager.in_memory_pages().contains_key(&2),
"Page 2 should still be cached because it was touched"
);
assert!(
!pager.in_memory_pages().contains_key(&3),
"Page 3 should be evicted as LRU"
);
assert!(
pager.in_memory_pages().contains_key(&5),
"Page 5 should be in cache"
);
let p3_reloaded = pager.read_page(3).unwrap();
assert_eq!(p3_reloaded[0], 3);
assert!(pager.in_memory_pages().contains_key(&3));
}
let _ = std::fs::remove_file(&db_path);
let _ = std::fs::remove_file(db_path.with_extension("tapir-wal"));
}
}