use std::io;
use std::ops::Range;
use std::sync::Arc;
use crate::io::engine::BoundedIoRequest;
use crate::io::engine::IoBuffer;
use crate::io::engine::IoCompletion;
use crate::io::engine::IoDeadlineExceeded;
use crate::io::engine::IoEngine;
use crate::io::engine::IoOperation;
use crate::io::engine::OperationKind;
use crate::io::engine::ReadSlot;
use crate::io::engine::RequestId;
use crate::io::engine::submit_cache_read;
use crate::region::index::packed::IndexEntry;
use crate::region::record::RECORD_ALIGNMENT;
use crate::region::recovery::DATA_REGION_AREA_OFFSET;
use crate::region::recovery::DataGeometry;
use crate::resources::BufferLease;
const _READ_ALIGNMENT: usize = 4096;
const _MAX_READ_ALIGNMENT_OVERHEAD: usize = 2 * _READ_ALIGNMENT;
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct ReadCandidate {
pub entry: IndexEntry,
pub region_generation: u64,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct ReadPlan {
pub hash: u64,
pub entry: IndexEntry,
pub region_generation: u64,
pub absolute: u64,
pub read_len: usize,
pub record_range: Range<usize>,
}
pub struct PendingRead {
plan: ReadPlan,
request_id: RequestId,
request: BoundedIoRequest,
}
pub struct ReadCompletion {
pub plan: ReadPlan,
pub result: io::Result<()>,
pub buffer: Option<BufferLease>,
}
impl ReadCompletion {
pub fn record_bytes(&self) -> Option<&[u8]> {
if self.result.is_err() {
return None;
}
self.buffer
.as_ref()?
.prepared(self.plan.read_len)
.ok()?
.get(self.plan.record_range.clone())
}
}
impl PendingRead {
#[cfg(test)]
pub fn wait(self, engine: &dyn IoEngine) -> ReadCompletion {
let Self {
plan,
request_id,
request,
} = self;
let completion = request.wait(engine);
Self::finish(plan, request_id, completion)
}
pub async fn wait_async(
self,
engine: Arc<dyn IoEngine>,
tokio_handle: &tokio::runtime::Handle,
) -> ReadCompletion {
let Self {
plan,
request_id,
request,
} = self;
let completion = request.wait_async(engine, tokio_handle).await;
Self::finish(plan, request_id, completion)
}
fn finish(
plan: ReadPlan,
request_id: RequestId,
completion: Result<IoCompletion, IoDeadlineExceeded>,
) -> ReadCompletion {
let completion = match completion {
Ok(completion) => completion,
Err(timeout) => {
let (error, buffer) = timeout.into_lease();
return ReadCompletion {
plan,
result: Err(error),
buffer,
};
}
};
let identity_valid =
completion.request_id == request_id && completion.kind == OperationKind::Read;
let bytes_transferred = completion.bytes_transferred;
let (io_result, buffer) = completion.into_lease();
let protocol_error = if !identity_valid {
Some(io::Error::new(
io::ErrorKind::InvalidData,
"Region record completion identity does not match its request",
))
} else if buffer.is_none() {
Some(io::Error::new(
io::ErrorKind::InvalidData,
"Region record completion did not return its owned buffer",
))
} else if buffer
.as_ref()
.is_some_and(|buffer| buffer.prepared(plan.read_len).is_err())
{
Some(io::Error::new(
io::ErrorKind::InvalidData,
"Region record completion returned a short buffer",
))
} else {
None
};
let result = match protocol_error {
Some(error) => Err(error),
None => io_result.and_then(|completed| {
if completed != plan.read_len || bytes_transferred != plan.read_len {
return Err(io::Error::new(
io::ErrorKind::UnexpectedEof,
"Region record read completed with the wrong byte count",
));
}
Ok(())
}),
};
ReadCompletion {
plan,
result,
buffer,
}
}
}
pub fn submit_read(
engine: &dyn IoEngine,
slot: ReadSlot,
plan: ReadPlan,
buffer: BufferLease,
) -> io::Result<PendingRead> {
let buffer = IoBuffer::for_read(buffer, plan.read_len).map_err(|error| error.error)?;
let request = submit_cache_read(engine, slot, IoOperation::read(buffer, plan.absolute))
.map_err(|error| error.into_lease().0)?;
Ok(PendingRead {
plan,
request_id: request.id(),
request,
})
}
pub fn plan_read(
geometry: DataGeometry,
hash: u64,
candidate: ReadCandidate,
align_for_direct_io: bool,
) -> io::Result<ReadPlan> {
let ReadCandidate {
entry,
region_generation,
} = candidate;
let location = entry.location;
let offset = u64::from(location.offset());
let record_len_upper = usize::try_from(location.record_len()).map_err(|_| {
io::Error::new(
io::ErrorKind::InvalidInput,
"Region record length does not fit this platform",
)
})?;
let record_len_upper_u64 = u64::try_from(record_len_upper).map_err(|_| {
io::Error::new(
io::ErrorKind::InvalidInput,
"Region record length does not fit the data geometry",
)
})?;
if !geometry.is_valid()
|| region_generation == 0
|| location.region_id() >= geometry.region_count
|| offset % u64::from(RECORD_ALIGNMENT) != 0
|| offset >= geometry.region_size
|| record_len_upper == 0
|| record_len_upper % RECORD_ALIGNMENT as usize != 0
{
return Err(io::Error::new(
io::ErrorKind::InvalidInput,
"invalid durable Region record location",
));
}
let record_len_u64 = record_len_upper_u64.min(geometry.region_size - offset);
let record_len = usize::try_from(record_len_u64).map_err(|_| {
io::Error::new(
io::ErrorKind::InvalidInput,
"bounded Region record length does not fit this platform",
)
})?;
let record_absolute = u64::from(location.region_id())
.checked_mul(geometry.region_size)
.and_then(|base| DATA_REGION_AREA_OFFSET.checked_add(base))
.and_then(|base| base.checked_add(offset))
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "read offset overflow"))?;
let record_absolute_end = record_absolute
.checked_add(record_len_u64)
.filter(|end| *end <= geometry.data_file_len)
.ok_or_else(|| {
io::Error::new(
io::ErrorKind::InvalidInput,
"Region record exceeds the data file",
)
})?;
let alignment = _READ_ALIGNMENT as u64;
let (absolute, io_end) = if align_for_direct_io {
(
record_absolute / alignment * alignment,
align_up(record_absolute_end, alignment).ok_or_else(|| {
io::Error::new(io::ErrorKind::InvalidInput, "aligned read overflow")
})?,
)
} else {
(record_absolute, record_absolute_end)
};
let read_len = io_end
.checked_sub(absolute)
.and_then(|length| usize::try_from(length).ok())
.filter(|length| *length != 0 && (!align_for_direct_io || *length % _READ_ALIGNMENT == 0))
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "invalid Region record read"))?;
let record_start = record_absolute
.checked_sub(absolute)
.and_then(|start| usize::try_from(start).ok())
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "record slice overflow"))?;
let record_range = record_start
..record_start.checked_add(record_len).ok_or_else(|| {
io::Error::new(io::ErrorKind::InvalidInput, "record slice end overflow")
})?;
let overhead = read_len.checked_sub(record_len).ok_or_else(|| {
io::Error::new(
io::ErrorKind::InvalidInput,
"Region read is shorter than its candidate range",
)
})?;
if record_range.end > read_len
|| (align_for_direct_io
&& (!absolute.is_multiple_of(alignment) || overhead >= _MAX_READ_ALIGNMENT_OVERHEAD))
|| io_end > geometry.data_file_len
{
return Err(io::Error::new(
io::ErrorKind::InvalidInput,
"Region read exceeds its bounds",
));
}
Ok(ReadPlan {
hash,
entry,
region_generation,
absolute,
read_len,
record_range,
})
}
fn align_up(value: u64, alignment: u64) -> Option<u64> {
value
.checked_add(alignment.checked_sub(1)?)
.map(|end| end / alignment * alignment)
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
use std::sync::Mutex;
use super::*;
use crate::io::backend::IoBackend;
use crate::io::backend::SyncMode;
use crate::io::backend::SyncPoint;
use crate::io::backend::WritePoint;
use crate::io::engine::BackendIoEngine;
use crate::region::index::packed::PackedLocation;
use crate::resources::ResourceController;
use crate::resources::ResourceLimits;
#[derive(Default)]
struct RecordingBackend {
reads: Mutex<Vec<(u64, usize)>>,
}
impl IoBackend for RecordingBackend {
fn len(&self) -> io::Result<u64> {
Ok(u64::MAX)
}
fn set_len(&self, _len: u64) -> io::Result<()> {
Ok(())
}
fn read_at(&self, buffer: &mut [u8], offset: u64) -> io::Result<usize> {
self.reads
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.push((offset, buffer.len()));
for (index, byte) in buffer.iter_mut().enumerate() {
*byte = offset.wrapping_add(index as u64) as u8;
}
Ok(buffer.len())
}
fn write_at(&self, _point: WritePoint, _buffer: &[u8], _offset: u64) -> io::Result<usize> {
Err(io::Error::new(io::ErrorKind::Unsupported, "write unused"))
}
fn sync(&self, _point: SyncPoint, _mode: SyncMode) -> io::Result<()> {
Ok(())
}
fn try_lock_exclusive(&self) -> io::Result<()> {
Ok(())
}
fn unlock(&self) -> io::Result<()> {
Ok(())
}
}
fn geometry() -> DataGeometry {
let region_size = 64 * 1024;
let region_count = 2;
DataGeometry {
data_file_len: DataGeometry::expected_file_len(region_size, region_count).unwrap(),
region_size,
region_count,
}
}
fn entry(location: PackedLocation) -> IndexEntry {
IndexEntry { location }
}
fn candidate(entry: IndexEntry) -> ReadCandidate {
ReadCandidate {
entry,
region_generation: 1,
}
}
#[test]
fn unaligned_record_uses_one_aligned_read_and_returns_its_exact_slice() {
let backend = Arc::new(RecordingBackend::default());
let engine = BackendIoEngine::new(backend.clone(), 1).unwrap();
let resources = ResourceController::try_new(ResourceLimits {
memory_limit_bytes: _READ_ALIGNMENT,
reserved_memory_bytes: 0,
})
.unwrap();
let location = PackedLocation::new(1, 32, 64).unwrap();
let entry = entry(location);
let plan = plan_read(geometry(), 7, candidate(entry), true).unwrap();
let slot = engine.try_reserve_read().unwrap();
let completion = submit_read(
&engine,
slot,
plan,
resources.try_read_buffer(_READ_ALIGNMENT).unwrap(),
)
.unwrap()
.wait(&engine);
assert!(completion.result.is_ok());
assert_eq!(completion.plan.record_range, 32..96);
assert_eq!(completion.record_bytes().unwrap().len(), 64);
let record_absolute = DATA_REGION_AREA_OFFSET + geometry().region_size + 32;
let reads = backend
.reads
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
assert_eq!(reads.as_slice(), &[(record_absolute - 32, 4096)]);
let expected = (0..64)
.map(|index| record_absolute.wrapping_add(index) as u8)
.collect::<Vec<_>>();
assert_eq!(completion.record_bytes().unwrap(), expected);
drop(reads);
drop(completion.buffer);
engine.shutdown().unwrap();
assert_eq!(resources.managed_memory_snapshot().current_bytes, 0);
}
#[test]
fn buffered_record_uses_one_size_class_upper_bound_read() {
let backend = Arc::new(RecordingBackend::default());
let engine = BackendIoEngine::new(backend.clone(), 1).unwrap();
let resources = ResourceController::try_new(ResourceLimits {
memory_limit_bytes: _READ_ALIGNMENT,
reserved_memory_bytes: 0,
})
.unwrap();
let entry = entry(PackedLocation::new(1, 32, 1120).unwrap());
let plan = plan_read(geometry(), 7, candidate(entry), false).unwrap();
let record_absolute = DATA_REGION_AREA_OFFSET + geometry().region_size + 32;
let completion = submit_read(
&engine,
engine.try_reserve_read().unwrap(),
plan,
resources.try_read_buffer(1120).unwrap(),
)
.unwrap()
.wait(&engine);
assert!(completion.result.is_ok());
assert_eq!(completion.plan.record_range, 0..1120);
assert_eq!(completion.record_bytes().unwrap().len(), 1120);
assert_eq!(
backend
.reads
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.as_slice(),
&[(record_absolute, 1120)]
);
drop(completion.buffer);
engine.shutdown().unwrap();
assert_eq!(resources.managed_memory_snapshot().current_bytes, 0);
}
#[test]
fn final_record_size_class_is_clamped_to_its_region() {
let record_len = 1056;
let offset = geometry().region_size as u32 - record_len;
let entry = entry(PackedLocation::new(1, offset, 1120).unwrap());
let plan = plan_read(geometry(), 7, candidate(entry), false).unwrap();
assert_eq!(plan.record_range, 0..record_len as usize);
assert_eq!(plan.read_len, record_len as usize);
assert_eq!(
plan.absolute,
DATA_REGION_AREA_OFFSET + geometry().region_size + u64::from(offset)
);
}
#[test]
fn invalid_entry_is_rejected_before_allocating_or_issuing_io() {
let backend = Arc::new(RecordingBackend::default());
let engine = BackendIoEngine::new(backend.clone(), 1).unwrap();
let invalid = entry(PackedLocation::new(geometry().region_count, 0, 32).unwrap());
let error = plan_read(geometry(), 7, candidate(invalid), true).unwrap_err();
assert_eq!(error.kind(), io::ErrorKind::InvalidInput);
assert!(
backend
.reads
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.is_empty()
);
let generation_zero = ReadCandidate {
entry: entry(PackedLocation::new(0, 0, 32).unwrap()),
region_generation: 0,
};
assert_eq!(
plan_read(geometry(), 7, generation_zero, true)
.unwrap_err()
.kind(),
io::ErrorKind::InvalidInput
);
engine.shutdown().unwrap();
}
}