#[cfg(test)]
use std::path::PathBuf;
#[cfg(test)]
use std::sync::{LazyLock, Mutex};
use std::{
fs::File,
io::{BufReader, ErrorKind, Read},
path::Path,
};
use shardline_index::{
FileChunkRecord, FileRecord, FileRecordInvariantError, FileRecordStorageLayout,
};
use shardline_protocol::ByteRange;
pub use shardline_server_core::ServerObjectStore;
pub use shardline_server_core::ServerObjectStoreError;
use shardline_storage::{ObjectKey, ObjectMetadata, ObjectPrefix, ObjectStore};
use crate::error::{IndexError, ObjectStoreError};
use crate::{
ObjectStorageAdapter, ServerConfig, ServerError, ServerFrontend, chunk_store::chunk_object_key,
server_frontend::append_referenced_term_bytes,
};
#[cfg(test)]
type LocalObjectReadHook = Box<dyn FnOnce() + Send>;
#[cfg(test)]
struct LocalObjectReadHookRegistration {
path: PathBuf,
hook: LocalObjectReadHook,
}
#[cfg(test)]
type LocalObjectReadHookSlot = Option<LocalObjectReadHookRegistration>;
#[cfg(test)]
static BEFORE_LOCAL_OBJECT_READ_HOOK: LazyLock<Mutex<LocalObjectReadHookSlot>> =
LazyLock::new(|| Mutex::new(None));
impl From<ServerObjectStoreError> for ServerError {
fn from(value: ServerObjectStoreError) -> Self {
match value {
ServerObjectStoreError::NotFound => Self::NotFound,
ServerObjectStoreError::Overflow => Self::Overflow,
ServerObjectStoreError::InvalidContentHash => Self::InvalidContentHash,
ServerObjectStoreError::StoredObjectLengthMismatch => {
Self::ObjectStore(ObjectStoreError::StoredLengthMismatch)
}
ServerObjectStoreError::Local(e) => Self::ObjectStore(ObjectStoreError::Local(e)),
ServerObjectStoreError::S3(e) => Self::ObjectStore(ObjectStoreError::S3(e)),
ServerObjectStoreError::Io(e) => Self::Io(e),
ServerObjectStoreError::NumericConversion(e) => Self::NumericConversion(e),
}
}
}
pub(crate) fn visit_object_prefix(
object_store: &ServerObjectStore,
prefix: &ObjectPrefix,
mut visitor: impl FnMut(ObjectMetadata) -> Result<(), ServerError>,
) -> Result<(), ServerError> {
object_store.visit_prefix(prefix, &mut visitor)
}
pub(crate) fn read_full_object(
object_store: &ServerObjectStore,
object_key: &ObjectKey,
length: u64,
) -> Result<Vec<u8>, ServerError> {
const MAX_FULL_OBJECT_READ_BYTES: u64 = 1_073_741_824;
if length > MAX_FULL_OBJECT_READ_BYTES {
return Err(ServerError::RequestBodyTooLarge);
}
if length == 0 {
return Ok(Vec::new());
}
if let ServerObjectStore::Local(store) = object_store {
let path = store.path_for_key(object_key);
let file = store.open_object_file(object_key)?;
let actual_length = validate_local_object_length(&file, length)?;
return read_open_local_object(&path, file, actual_length);
}
let end = length.checked_sub(1).ok_or(ServerError::Overflow)?;
let range = ByteRange::new(0, end).map_err(|_error| ServerError::Overflow)?;
Ok(object_store.read_range(object_key, range)?)
}
pub(crate) fn reconstruct_local_file_bytes(
object_store: &ServerObjectStore,
chunks: &[FileChunkRecord],
capacity: usize,
) -> Result<Vec<u8>, ServerError> {
let mut output = Vec::with_capacity(capacity);
for chunk in chunks {
let expected_offset = u64::try_from(output.len())?;
if chunk.offset != expected_offset {
return Err(ServerError::Index(IndexError::FileRecordInvariant(
FileRecordInvariantError::NonContiguousChunkOffsets,
)));
}
let object_key = chunk_object_key(&chunk.hash)?;
let ServerObjectStore::Local(store) = object_store else {
return Err(ServerError::NotFound);
};
let path = store.path_for_key(&object_key);
let file = store.open_object_file(&object_key)?;
read_open_local_object_append(&path, file, chunk.length, &mut output)?;
}
if output.len() != capacity {
return Err(ServerError::ObjectStore(
ObjectStoreError::StoredLengthMismatch,
));
}
Ok(output)
}
pub(crate) fn reconstruct_file_record_bytes(
object_store: &ServerObjectStore,
frontends: &[ServerFrontend],
record: &FileRecord,
) -> Result<Vec<u8>, ServerError> {
let capacity = usize::try_from(record.total_bytes)?;
match record.storage_layout() {
FileRecordStorageLayout::ReferencedObjectTerms => {
reconstruct_referenced_object_file_bytes(object_store, frontends, record, capacity)
}
FileRecordStorageLayout::StoredChunks => {
reconstruct_chunk_file_bytes(object_store, &record.chunks, capacity)
}
}
}
fn reconstruct_chunk_file_bytes(
object_store: &ServerObjectStore,
chunks: &[FileChunkRecord],
capacity: usize,
) -> Result<Vec<u8>, ServerError> {
if object_store.local_root().is_some() {
return reconstruct_local_file_bytes(object_store, chunks, capacity);
}
let mut output = Vec::with_capacity(capacity);
for chunk in chunks {
let object_key = chunk_object_key(&chunk.hash)?;
let bytes = read_full_object(object_store, &object_key, chunk.length)?;
output.extend_from_slice(&bytes);
}
if output.len() != capacity {
return Err(ServerError::ObjectStore(
ObjectStoreError::StoredLengthMismatch,
));
}
Ok(output)
}
fn reconstruct_referenced_object_file_bytes(
object_store: &ServerObjectStore,
frontends: &[ServerFrontend],
record: &FileRecord,
capacity: usize,
) -> Result<Vec<u8>, ServerError> {
let mut output = Vec::with_capacity(capacity);
for term in &record.chunks {
let term_start = u64::try_from(output.len())?;
if term.offset != term_start {
return Err(ServerError::Index(IndexError::FileRecordInvariant(
FileRecordInvariantError::NonContiguousChunkOffsets,
)));
}
append_referenced_term_bytes(frontends, object_store, term, &mut output)?;
let term_end = term
.offset
.checked_add(term.length)
.ok_or(ServerError::Overflow)?;
if u64::try_from(output.len())? != term_end {
return Err(ServerError::ObjectStore(
ObjectStoreError::StoredLengthMismatch,
));
}
}
if output.len() != capacity {
return Err(ServerError::ObjectStore(
ObjectStoreError::StoredLengthMismatch,
));
}
Ok(output)
}
fn read_open_local_object(path: &Path, file: File, length: u64) -> Result<Vec<u8>, ServerError> {
let capacity = usize::try_from(length)?;
let mut output = Vec::with_capacity(capacity);
read_open_local_object_append(path, file, length, &mut output)?;
if output.len() != capacity {
return Err(ServerError::ObjectStore(
ObjectStoreError::StoredLengthMismatch,
));
}
Ok(output)
}
fn read_open_local_object_append(
path: &Path,
file: File,
expected_length: u64,
output: &mut Vec<u8>,
) -> Result<(), ServerError> {
validate_local_object_length(&file, expected_length)?;
run_before_local_object_read_hook(path);
let start_len = output.len();
let mut reader = BufReader::new(file);
let mut limited = reader.by_ref().take(expected_length);
if let Err(error) = limited.read_to_end(output) {
if error.kind() == ErrorKind::UnexpectedEof {
return Err(ServerError::ObjectStore(
ObjectStoreError::StoredLengthMismatch,
));
}
return Err(ServerError::Io(error));
}
let read_len = output
.len()
.checked_sub(start_len)
.ok_or(ServerError::Overflow)?;
if u64::try_from(read_len)? != expected_length {
return Err(ServerError::ObjectStore(
ObjectStoreError::StoredLengthMismatch,
));
}
let mut trailing_byte = [0_u8; 1];
match reader.read(&mut trailing_byte) {
Ok(0) => {}
Ok(_observed) => {
return Err(ServerError::ObjectStore(
ObjectStoreError::StoredLengthMismatch,
));
}
Err(error) if error.kind() == ErrorKind::UnexpectedEof => {
return Err(ServerError::ObjectStore(
ObjectStoreError::StoredLengthMismatch,
));
}
Err(error) => return Err(ServerError::Io(error)),
}
validate_local_object_length(&reader.into_inner(), expected_length)?;
Ok(())
}
fn validate_local_object_length(file: &File, expected_length: u64) -> Result<u64, ServerError> {
let actual_length = file.metadata()?.len();
if actual_length != expected_length {
return Err(ServerError::ObjectStore(
ObjectStoreError::StoredLengthMismatch,
));
}
Ok(actual_length)
}
#[cfg(test)]
pub(crate) fn run_before_local_object_read_hook(path: &Path) {
let hook = match BEFORE_LOCAL_OBJECT_READ_HOOK.lock() {
Ok(mut guard) => take_local_object_read_hook_for_path(&mut guard, path),
Err(poisoned) => take_local_object_read_hook_for_path(&mut poisoned.into_inner(), path),
};
if let Some(hook) = hook {
hook();
}
}
#[cfg(test)]
fn take_local_object_read_hook_for_path(
slot: &mut LocalObjectReadHookSlot,
path: &Path,
) -> Option<LocalObjectReadHook> {
if !matches!(slot, Some(registration) if registration.path == path) {
return None;
}
slot.take().map(|registration| registration.hook)
}
#[cfg(not(test))]
pub(crate) const fn run_before_local_object_read_hook(_path: &Path) {}
#[cfg(test)]
pub(crate) fn set_before_local_object_read_hook(
path: PathBuf,
hook: impl FnOnce() + Send + 'static,
) {
let mut slot = match BEFORE_LOCAL_OBJECT_READ_HOOK.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
};
*slot = Some(LocalObjectReadHookRegistration {
path,
hook: Box::new(hook),
});
}
pub(crate) fn object_store_from_config(
config: &ServerConfig,
) -> Result<ServerObjectStore, ServerError> {
match config.object_storage_adapter() {
ObjectStorageAdapter::Local => {
Ok(ServerObjectStore::local(config.root_dir().join("chunks"))?)
}
ObjectStorageAdapter::S3 => {
let s3_config = config
.s3_object_store_config()
.ok_or(ServerError::ObjectStore(ObjectStoreError::MissingS3Config))?
.clone();
Ok(ServerObjectStore::s3(s3_config)?)
}
}
}
#[cfg(test)]
mod tests {
use std::{
fs::{File, OpenOptions},
io::Write,
};
use shardline_index::{FileChunkRecord, FileRecord};
use shardline_storage::{ObjectKey, ObjectPrefix};
use super::{
read_full_object, read_open_local_object_append, reconstruct_chunk_file_bytes,
reconstruct_file_record_bytes, reconstruct_local_file_bytes,
set_before_local_object_read_hook, visit_object_prefix,
};
use crate::ServerConfig;
use crate::ServerError;
use crate::chunk_store::chunk_object_key;
use crate::error::ObjectStoreError;
use crate::object_store::ServerObjectStore;
#[test]
fn local_object_read_rejects_growth_after_length_validation_without_retaining_growth_bytes() {
let mut object = tempfile::NamedTempFile::new().unwrap();
object.write_all(b"abcd").unwrap();
object.as_file().sync_all().unwrap();
let reader = File::open(object.path()).unwrap();
let mut writer = OpenOptions::new().append(true).open(object.path()).unwrap();
set_before_local_object_read_hook(object.path().to_path_buf(), move || {
writer.write_all(b"efgh").unwrap();
writer.sync_all().unwrap();
});
let mut output = Vec::new();
let result = read_open_local_object_append(object.path(), reader, 4, &mut output);
assert!(matches!(
result,
Err(ServerError::ObjectStore(
ObjectStoreError::StoredLengthMismatch
))
));
assert_eq!(output, b"abcd");
}
#[test]
fn reconstruct_local_file_bytes_reassembles_chunks_correctly() {
let dir = tempfile::tempdir().unwrap();
let store = ServerObjectStore::local(dir.path()).unwrap();
let chunk1_hash = "aa".repeat(32); let chunk2_hash = "bb".repeat(32);
let chunk1_data = b"hello ";
let chunk2_data = b"world";
let key1 = chunk_object_key(&chunk1_hash).unwrap();
let key2 = chunk_object_key(&chunk2_hash).unwrap();
let path1 = dir.path().join(key1.as_str());
let path2 = dir.path().join(key2.as_str());
std::fs::create_dir_all(path1.parent().unwrap()).unwrap();
std::fs::create_dir_all(path2.parent().unwrap()).unwrap();
std::fs::write(&path1, chunk1_data).unwrap();
std::fs::write(&path2, chunk2_data).unwrap();
let chunks = vec![
FileChunkRecord {
hash: chunk1_hash,
offset: 0,
length: 6,
range_start: 0,
range_end: 1,
packed_start: 0,
packed_end: 6,
},
FileChunkRecord {
hash: chunk2_hash,
offset: 6,
length: 5,
range_start: 0,
range_end: 1,
packed_start: 0,
packed_end: 5,
},
];
let result = reconstruct_local_file_bytes(&store, &chunks, 11);
assert!(result.is_ok());
assert_eq!(result.unwrap(), b"hello world");
}
#[test]
fn reconstruct_local_file_bytes_rejects_non_contiguous_offsets() {
let dir = tempfile::tempdir().unwrap();
let store = ServerObjectStore::local(dir.path()).unwrap();
let hash = "cc".repeat(32);
let key = chunk_object_key(&hash).unwrap();
let path = dir.path().join(key.as_str());
std::fs::create_dir_all(path.parent().unwrap()).unwrap();
std::fs::write(&path, b"data").unwrap();
let chunks = vec![FileChunkRecord {
hash,
offset: 10, length: 4,
range_start: 0,
range_end: 1,
packed_start: 0,
packed_end: 4,
}];
let result = reconstruct_local_file_bytes(&store, &chunks, 4);
assert!(result.is_err());
}
#[test]
fn reconstruct_local_file_bytes_rejects_capacity_mismatch() {
let dir = tempfile::tempdir().unwrap();
let store = ServerObjectStore::local(dir.path()).unwrap();
let hash = "dd".repeat(32);
let key = chunk_object_key(&hash).unwrap();
let path = dir.path().join(key.as_str());
std::fs::create_dir_all(path.parent().unwrap()).unwrap();
std::fs::write(&path, b"data").unwrap();
let chunks = vec![FileChunkRecord {
hash,
offset: 0,
length: 4,
range_start: 0,
range_end: 1,
packed_start: 0,
packed_end: 4,
}];
let result = reconstruct_local_file_bytes(&store, &chunks, 99);
assert!(matches!(
result,
Err(ServerError::ObjectStore(
ObjectStoreError::StoredLengthMismatch
))
));
}
#[test]
fn reconstruct_local_file_bytes_fails_with_blackhole_store() {
let store = ServerObjectStore::blackhole();
let hash = "ff".repeat(32);
let chunks = vec![FileChunkRecord {
hash,
offset: 0,
length: 4,
range_start: 0,
range_end: 1,
packed_start: 0,
packed_end: 4,
}];
let result = reconstruct_local_file_bytes(&store, &chunks, 4);
assert!(matches!(result, Err(ServerError::NotFound)));
}
#[test]
fn read_open_local_object_append_rejects_trailing_bytes() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("trailing.bin");
std::fs::write(&path, b"abcdef").unwrap();
let file = File::open(&path).unwrap();
let mut output = Vec::new();
let result = read_open_local_object_append(&path, file, 4, &mut output);
assert!(
matches!(
result,
Err(ServerError::ObjectStore(
ObjectStoreError::StoredLengthMismatch
))
),
"expected StoredLengthMismatch for trailing bytes, got {result:?}"
);
}
#[test]
fn read_open_local_object_append_rejects_shorter_file() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("short.bin");
std::fs::write(&path, b"abc").unwrap();
let file = File::open(&path).unwrap();
let mut output = Vec::new();
let result = read_open_local_object_append(&path, file, 5, &mut output);
assert!(matches!(
result,
Err(ServerError::ObjectStore(
ObjectStoreError::StoredLengthMismatch
))
));
}
#[test]
fn reconstruct_chunk_file_bytes_with_local_store() {
let dir = tempfile::tempdir().unwrap();
let store = ServerObjectStore::local(dir.path()).unwrap();
let hash = "ee".repeat(32);
let key = chunk_object_key(&hash).unwrap();
let path = dir.path().join(key.as_str());
std::fs::create_dir_all(path.parent().unwrap()).unwrap();
std::fs::write(&path, b"chunk-data").unwrap();
let chunks = vec![FileChunkRecord {
hash,
offset: 0,
length: 10,
range_start: 0,
range_end: 1,
packed_start: 0,
packed_end: 10,
}];
let result = reconstruct_chunk_file_bytes(&store, &chunks, 10);
assert!(result.is_ok());
assert_eq!(result.unwrap(), b"chunk-data");
}
#[test]
fn reconstruct_chunk_file_bytes_empty_chunks_with_local_store() {
let dir = tempfile::tempdir().unwrap();
let store = ServerObjectStore::local(dir.path()).unwrap();
let result = reconstruct_chunk_file_bytes(&store, &[], 0);
assert!(result.is_ok());
assert_eq!(result.unwrap(), b"");
}
#[test]
fn reconstruct_chunk_file_bytes_blackhole_errors_on_nonzero_length() {
let store = ServerObjectStore::blackhole();
let hash = "ff".repeat(32);
let chunks = vec![FileChunkRecord {
hash,
offset: 0,
length: 4,
range_start: 0,
range_end: 1,
packed_start: 0,
packed_end: 4,
}];
let result = reconstruct_chunk_file_bytes(&store, &chunks, 4);
assert!(matches!(result, Err(ServerError::NotFound)));
}
#[test]
fn reconstruct_file_record_bytes_with_stored_chunks_layout() {
let dir = tempfile::tempdir().unwrap();
let store = ServerObjectStore::local(dir.path()).unwrap();
let data = b"stored-bytes";
let hash = "11".repeat(32);
let key = chunk_object_key(&hash).unwrap();
let path = dir.path().join(key.as_str());
std::fs::create_dir_all(path.parent().unwrap()).unwrap();
std::fs::write(&path, data).unwrap();
let file_len = std::fs::metadata(&path).unwrap().len();
assert_eq!(file_len, data.len() as u64, "written file length mismatch");
let record = FileRecord {
file_id: "test.bin".to_owned(),
content_hash: "aa".repeat(32),
total_bytes: data.len() as u64,
chunk_size: data.len() as u64, repository_scope: None,
chunks: vec![FileChunkRecord {
hash,
offset: 0,
length: data.len() as u64,
range_start: 0,
range_end: 1,
packed_start: 0,
packed_end: data.len() as u64,
}],
};
let result = reconstruct_file_record_bytes(&store, &[], &record);
assert!(
result.is_ok(),
"reconstruct_file_record_bytes failed: {:?}",
result.as_ref().err()
);
assert_eq!(result.unwrap(), data);
}
#[test]
fn reconstruct_file_record_bytes_referenced_terms_rejects_empty_frontends() {
let dir = tempfile::tempdir().unwrap();
let store = ServerObjectStore::local(dir.path()).unwrap();
let record = FileRecord {
file_id: "ref.bin".to_owned(),
content_hash: "bb".repeat(32),
total_bytes: 4,
chunk_size: 0,
repository_scope: None,
chunks: vec![FileChunkRecord {
hash: "cc".repeat(32),
offset: 0,
length: 4,
range_start: 0,
range_end: 1,
packed_start: 0,
packed_end: 4,
}],
};
let result = reconstruct_file_record_bytes(&store, &[], &record);
assert!(result.is_err());
}
#[test]
fn visit_object_prefix_with_blackhole_returns_ok() {
let store = ServerObjectStore::blackhole();
let prefix = ObjectPrefix::parse("test-prefix").unwrap();
let result = visit_object_prefix(&store, &prefix, |_metadata| Ok(()));
assert!(result.is_ok());
}
#[test]
fn read_full_object_zero_length_returns_empty() {
let store = ServerObjectStore::blackhole();
let key = ObjectKey::parse("test/key").unwrap();
let result = read_full_object(&store, &key, 0);
assert!(result.is_ok());
assert_eq!(result.unwrap(), b"");
}
#[test]
fn read_full_object_blackhole_nonzero_returns_not_found() {
let store = ServerObjectStore::blackhole();
let key = ObjectKey::parse("test/key").unwrap();
let result = read_full_object(&store, &key, 10);
assert!(matches!(result, Err(ServerError::NotFound)));
}
#[test]
fn read_full_object_exceeds_max_returns_too_large() {
let store = ServerObjectStore::blackhole();
let key = ObjectKey::parse("test/key").unwrap();
let result = read_full_object(&store, &key, 2_000_000_000);
assert!(matches!(result, Err(ServerError::RequestBodyTooLarge)));
}
#[test]
fn read_full_object_local_store_reads_file() {
let dir = tempfile::tempdir().unwrap();
let store = ServerObjectStore::local(dir.path()).unwrap();
let key = ObjectKey::parse("my-object").unwrap();
let object_path = dir.path().join(key.as_str());
if let Some(parent) = object_path.parent() {
std::fs::create_dir_all(parent).unwrap();
}
let data = b"full-object-data";
std::fs::write(&object_path, data).unwrap();
let data_len = data.len() as u64;
let result = read_full_object(&store, &key, data_len);
assert!(
result.is_ok(),
"read_full_object failed: {:?}",
result.as_ref().err()
);
assert_eq!(result.unwrap(), data);
}
#[test]
fn server_object_store_error_not_found_converts_to_not_found() {
let err: ServerError = shardline_server_core::ServerObjectStoreError::NotFound.into();
assert!(matches!(err, ServerError::NotFound));
}
#[test]
fn server_object_store_error_overflow_converts_to_overflow() {
let err: ServerError = shardline_server_core::ServerObjectStoreError::Overflow.into();
assert!(matches!(err, ServerError::Overflow));
}
#[test]
fn server_object_store_error_invalid_content_hash_converts() {
let err: ServerError =
shardline_server_core::ServerObjectStoreError::InvalidContentHash.into();
assert!(matches!(err, ServerError::InvalidContentHash));
}
#[test]
fn object_store_from_config_local_adapter_returns_local_store() {
let tmp = tempfile::tempdir().unwrap();
let config = ServerConfig::new(
std::net::SocketAddr::new(std::net::IpAddr::V4(std::net::Ipv4Addr::LOCALHOST), 8080),
"http://127.0.0.1:8080".to_owned(),
tmp.path().to_path_buf(),
std::num::NonZeroUsize::new(65536).unwrap_or(std::num::NonZeroUsize::MIN),
);
let store = super::object_store_from_config(&config);
assert!(store.is_ok());
assert!(store.unwrap().local_root().is_some());
}
#[test]
fn object_store_from_config_s3_without_config_returns_error() {
let tmp = tempfile::tempdir().unwrap();
let config = ServerConfig::new(
std::net::SocketAddr::new(std::net::IpAddr::V4(std::net::Ipv4Addr::LOCALHOST), 8080),
"http://127.0.0.1:8080".to_owned(),
tmp.path().to_path_buf(),
std::num::NonZeroUsize::new(65536).unwrap_or(std::num::NonZeroUsize::MIN),
)
.with_object_storage(crate::ObjectStorageAdapter::S3, None);
let store = super::object_store_from_config(&config);
assert!(matches!(
store,
Err(ServerError::ObjectStore(ObjectStoreError::MissingS3Config))
));
}
#[test]
fn server_object_store_error_stored_length_mismatch_converts() {
let err: ServerError =
shardline_server_core::ServerObjectStoreError::StoredObjectLengthMismatch.into();
assert!(matches!(
err,
ServerError::ObjectStore(ObjectStoreError::StoredLengthMismatch)
));
}
#[test]
fn server_object_store_error_local_converts() {
let io_err = std::io::Error::other("test");
let err: ServerError =
shardline_server_core::ServerObjectStoreError::Local(io_err.into()).into();
assert!(matches!(
err,
ServerError::ObjectStore(ObjectStoreError::Local(_))
));
}
#[test]
fn read_open_local_object_reads_exact_content() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("exact.bin");
let data = b"exact-length-content";
std::fs::write(&path, data).unwrap();
let file = std::fs::File::open(&path).unwrap();
let result = super::read_open_local_object(&path, file, data.len() as u64);
assert!(result.is_ok());
assert_eq!(result.unwrap(), data);
}
#[test]
fn read_open_local_object_rejects_trailing_bytes() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("trailing2.bin");
std::fs::write(&path, b"abcdefgh").unwrap();
let file = std::fs::File::open(&path).unwrap();
let result = super::read_open_local_object(&path, file, 5);
assert!(matches!(
result,
Err(ServerError::ObjectStore(
ObjectStoreError::StoredLengthMismatch
))
));
}
#[test]
fn read_open_local_object_rejects_shorter_file() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("short2.bin");
std::fs::write(&path, b"abc").unwrap();
let file = std::fs::File::open(&path).unwrap();
let result = super::read_open_local_object(&path, file, 10);
assert!(matches!(
result,
Err(ServerError::ObjectStore(
ObjectStoreError::StoredLengthMismatch
))
));
}
#[test]
fn validate_local_object_length_exact_match() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("exact_len.bin");
std::fs::write(&path, b"12345").unwrap();
let file = std::fs::File::open(&path).unwrap();
let result = super::validate_local_object_length(&file, 5);
assert!(result.is_ok());
assert_eq!(result.unwrap(), 5);
}
#[test]
fn validate_local_object_length_mismatch() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("mismatch_len.bin");
std::fs::write(&path, b"12345").unwrap();
let file = std::fs::File::open(&path).unwrap();
let result = super::validate_local_object_length(&file, 10);
assert!(matches!(
result,
Err(ServerError::ObjectStore(
ObjectStoreError::StoredLengthMismatch
))
));
}
#[test]
fn reconstruct_referenced_object_file_bytes_rejects_non_contiguous_offsets() {
let dir = tempfile::tempdir().unwrap();
let store = ServerObjectStore::local(dir.path()).unwrap();
let record = shardline_index::FileRecord {
file_id: "ref.bin".to_owned(),
content_hash: "aa".repeat(32),
total_bytes: 10,
chunk_size: 0, repository_scope: None,
chunks: vec![shardline_index::FileChunkRecord {
hash: "cc".repeat(32),
offset: 10, length: 4,
range_start: 0,
range_end: 1,
packed_start: 0,
packed_end: 4,
}],
};
let result = super::reconstruct_file_record_bytes(&store, &[], &record);
assert!(result.is_err());
}
#[test]
fn reconstruct_chunk_file_bytes_non_local_store_uses_read_full_object() {
let store = ServerObjectStore::blackhole();
let hash = "11".repeat(32);
let chunks = vec![shardline_index::FileChunkRecord {
hash,
offset: 0,
length: 4,
range_start: 0,
range_end: 1,
packed_start: 0,
packed_end: 4,
}];
let result = reconstruct_chunk_file_bytes(&store, &chunks, 4);
assert!(matches!(result, Err(ServerError::NotFound)));
}
#[test]
fn visit_object_prefix_with_local_store_lists_objects() {
let dir = tempfile::tempdir().unwrap();
let store = ServerObjectStore::local(dir.path()).unwrap();
let key = ObjectKey::parse("prefix/obj").unwrap();
let path = dir.path().join(key.as_str());
std::fs::create_dir_all(path.parent().unwrap()).unwrap();
std::fs::write(&path, b"data").unwrap();
let prefix = ObjectPrefix::parse("prefix").unwrap();
let mut count = 0;
visit_object_prefix(&store, &prefix, |_meta| {
count += 1;
Ok(())
})
.unwrap();
assert_eq!(count, 1);
}
#[test]
fn object_store_from_config_s3_with_config_returns_s3_store() {
let tmp = tempfile::tempdir().unwrap();
let s3_config = shardline_storage::S3ObjectStoreConfig::new(
"bucket".to_owned(),
"us-east-1".to_owned(),
);
let config = ServerConfig::new(
std::net::SocketAddr::new(std::net::IpAddr::V4(std::net::Ipv4Addr::LOCALHOST), 8080),
"http://127.0.0.1:8080".to_owned(),
tmp.path().to_path_buf(),
std::num::NonZeroUsize::new(65536).unwrap_or(std::num::NonZeroUsize::MIN),
)
.with_object_storage(crate::ObjectStorageAdapter::S3, Some(s3_config));
let store = super::object_store_from_config(&config);
assert!(store.is_ok());
assert!(store.unwrap().local_root().is_none());
}
#[test]
fn read_full_object_local_store_rejects_length_mismatch() {
let dir = tempfile::tempdir().unwrap();
let store = ServerObjectStore::local(dir.path()).unwrap();
let key = ObjectKey::parse("mismatch-obj").unwrap();
let object_path = dir.path().join(key.as_str());
if let Some(parent) = object_path.parent() {
std::fs::create_dir_all(parent).unwrap();
}
std::fs::write(&object_path, b"short").unwrap();
let result = read_full_object(&store, &key, 100);
assert!(matches!(
result,
Err(ServerError::ObjectStore(
ObjectStoreError::StoredLengthMismatch
))
));
}
#[test]
fn read_full_object_local_store_zero_length_returns_empty() {
let dir = tempfile::tempdir().unwrap();
let store = ServerObjectStore::local(dir.path()).unwrap();
let key = ObjectKey::parse("empty-obj").unwrap();
let object_path = dir.path().join(key.as_str());
if let Some(parent) = object_path.parent() {
std::fs::create_dir_all(parent).unwrap();
}
std::fs::write(&object_path, b"").unwrap();
let result = read_full_object(&store, &key, 0);
assert!(result.is_ok());
assert_eq!(result.unwrap(), b"");
}
#[test]
fn take_local_object_read_hook_for_path_returns_none_for_mismatched_path() {
let result = super::take_local_object_read_hook_for_path(
&mut None,
std::path::Path::new("/nonexistent"),
);
assert!(result.is_none());
}
#[test]
fn read_open_local_object_append_read_to_end_eof_returns_length_mismatch() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("partial.bin");
std::fs::write(&path, b"short").unwrap();
let file = std::fs::File::open(&path).unwrap();
let mut output = Vec::new();
let result = read_open_local_object_append(&path, file, 100, &mut output);
assert!(matches!(
result,
Err(ServerError::ObjectStore(
ObjectStoreError::StoredLengthMismatch
))
));
}
#[test]
fn read_open_local_object_append_eof_on_trailing_byte_check_returns_length_mismatch() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("eof-trailing.bin");
std::fs::write(&path, b"exact").unwrap();
let file = std::fs::File::open(&path).unwrap();
let mut output = Vec::new();
let result = read_open_local_object_append(&path, file, 5, &mut output);
assert!(result.is_ok());
assert_eq!(output, b"exact");
}
#[test]
fn reconstruct_chunk_file_bytes_local_store_rejects_capacity_mismatch() {
let dir = tempfile::tempdir().unwrap();
let store = ServerObjectStore::local(dir.path()).unwrap();
let hash = "ca".repeat(32);
let key = chunk_object_key(&hash).unwrap();
let path = dir.path().join(key.as_str());
std::fs::create_dir_all(path.parent().unwrap()).unwrap();
std::fs::write(&path, b"data").unwrap();
let chunks = vec![FileChunkRecord {
hash,
offset: 0,
length: 4,
range_start: 0,
range_end: 1,
packed_start: 0,
packed_end: 4,
}];
let result = reconstruct_chunk_file_bytes(&store, &chunks, 99);
assert!(matches!(
result,
Err(ServerError::ObjectStore(
ObjectStoreError::StoredLengthMismatch
))
));
}
#[test]
fn reconstruct_referenced_object_file_bytes_rejects_capacity_mismatch() {
let dir = tempfile::tempdir().unwrap();
let store = ServerObjectStore::local(dir.path()).unwrap();
let record = shardline_index::FileRecord {
file_id: "cap.bin".to_owned(),
content_hash: "cc".repeat(32),
total_bytes: 99, chunk_size: 0,
repository_scope: None,
chunks: vec![shardline_index::FileChunkRecord {
hash: "dd".repeat(32),
offset: 0,
length: 4,
range_start: 0,
range_end: 1,
packed_start: 0,
packed_end: 4,
}],
};
let result = super::reconstruct_file_record_bytes(&store, &[], &record);
assert!(result.is_err());
}
#[test]
fn read_open_local_object_append_requires_exact_read_length() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("read-exact.bin");
std::fs::write(&path, b"toolong").unwrap();
let file = std::fs::File::open(&path).unwrap();
let mut output = Vec::new();
let result = read_open_local_object_append(&path, file, 4, &mut output);
assert!(matches!(
result,
Err(ServerError::ObjectStore(
ObjectStoreError::StoredLengthMismatch
))
));
}
#[test]
fn server_object_store_error_io_converts() {
let io_err = std::io::Error::other("disk error");
let err: ServerError = shardline_server_core::ServerObjectStoreError::Io(io_err).into();
assert!(matches!(err, ServerError::Io(_)));
}
#[test]
fn server_object_store_error_numeric_conversion_converts() {
let int_err = i32::try_from(3_000_000_000_i64).unwrap_err();
let err: ServerError =
shardline_server_core::ServerObjectStoreError::NumericConversion(int_err).into();
assert!(matches!(err, ServerError::NumericConversion(_)));
}
#[test]
fn read_full_object_local_store_short_read_returns_length_mismatch() {
let dir = tempfile::tempdir().unwrap();
let store = ServerObjectStore::local(dir.path()).unwrap();
let key = ObjectKey::parse("short-read").unwrap();
let object_path = dir.path().join(key.as_str());
if let Some(parent) = object_path.parent() {
std::fs::create_dir_all(parent).unwrap();
}
std::fs::write(&object_path, b"abc").unwrap();
let result = read_full_object(&store, &key, 10);
assert!(matches!(
result,
Err(ServerError::ObjectStore(
ObjectStoreError::StoredLengthMismatch
))
));
}
#[test]
fn read_open_local_object_append_returns_io_error_on_read_failure() {
let dir = tempfile::tempdir().unwrap();
let dir_path = dir.path().to_path_buf();
let dir_file = std::fs::File::open(&dir_path).unwrap();
let meta_len = dir_file.metadata().unwrap().len();
if meta_len == 0 {
return;
}
let mut output = Vec::new();
let result = read_open_local_object_append(&dir_path, dir_file, meta_len, &mut output);
assert!(
matches!(result, Err(ServerError::Io(_))),
"expected Io error when reading from a directory fd, got {result:?}"
);
}
#[test]
fn server_object_store_error_s3_converts() {
use shardline_storage::S3ObjectStoreError;
let s3_err =
shardline_server_core::ServerObjectStoreError::S3(S3ObjectStoreError::EmptyBucket);
let err: ServerError = s3_err.into();
assert!(matches!(
err,
ServerError::ObjectStore(ObjectStoreError::S3(_))
));
}
#[test]
fn read_full_object_length_one_overflow_edge() {
let store = ServerObjectStore::blackhole();
let key = ObjectKey::parse("test/edge").unwrap();
let result = read_full_object(&store, &key, 1);
assert!(matches!(result, Err(ServerError::NotFound)));
}
#[test]
fn reconstruct_chunk_file_bytes_non_local_rejects_capacity_mismatch() {
let store = ServerObjectStore::blackhole();
let hash = "11".repeat(32);
let chunks = vec![FileChunkRecord {
hash,
offset: 0,
length: 4,
range_start: 0,
range_end: 1,
packed_start: 0,
packed_end: 4,
}];
let result = reconstruct_chunk_file_bytes(&store, &chunks, 99);
assert!(matches!(result, Err(ServerError::NotFound)));
}
#[test]
fn run_before_local_object_read_hook_without_registration_returns_immediately() {
let path = std::path::Path::new("/nonexistent/path");
crate::object_store::run_before_local_object_read_hook(path);
}
#[test]
fn set_before_local_object_read_hook_overwrites_previous_registration() {
let path1 = std::path::PathBuf::from("/tmp/hook-overwrite-1");
let path2 = std::path::PathBuf::from("/tmp/hook-overwrite-2");
let called1 = std::sync::atomic::AtomicBool::new(false);
let called2 = std::sync::atomic::AtomicBool::new(false);
set_before_local_object_read_hook(path1, move || {
called1.store(true, std::sync::atomic::Ordering::SeqCst);
});
set_before_local_object_read_hook(path2.clone(), move || {
called2.store(true, std::sync::atomic::Ordering::SeqCst);
});
let slot = super::BEFORE_LOCAL_OBJECT_READ_HOOK.lock().unwrap();
assert!(slot.is_some());
let registration = slot.as_ref().unwrap();
assert_eq!(registration.path, path2);
}
#[test]
fn reconstruct_chunk_file_bytes_local_store_exact_capacity() {
let dir = tempfile::tempdir().unwrap();
let store = ServerObjectStore::local(dir.path()).unwrap();
let hash = "aa".repeat(32);
let data = b"exact-data";
let key = chunk_object_key(&hash).unwrap();
let path = dir.path().join(key.as_str());
std::fs::create_dir_all(path.parent().unwrap()).unwrap();
std::fs::write(&path, data).unwrap();
let chunks = vec![FileChunkRecord {
hash,
offset: 0,
length: data.len() as u64,
range_start: 0,
range_end: 1,
packed_start: 0,
packed_end: data.len() as u64,
}];
let result = reconstruct_chunk_file_bytes(&store, &chunks, data.len());
assert!(result.is_ok());
assert_eq!(result.unwrap(), data);
}
#[test]
fn read_open_local_object_append_read_len_mismatch_after_take() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("read-len-mismatch.bin");
std::fs::write(&path, b"exact-content").unwrap();
let file = std::fs::File::open(&path).unwrap();
let mut output = Vec::new();
let result =
crate::object_store::read_open_local_object_append(&path, file, 13, &mut output);
assert!(result.is_ok());
assert_eq!(output, b"exact-content");
}
#[test]
fn reconstruct_referenced_object_file_bytes_capacity_check() {
let dir = tempfile::tempdir().unwrap();
let store = ServerObjectStore::local(dir.path()).unwrap();
let record = shardline_index::FileRecord {
file_id: "cap-ref.bin".to_owned(),
content_hash: "dd".repeat(32),
total_bytes: 99,
chunk_size: 0,
repository_scope: None,
chunks: vec![shardline_index::FileChunkRecord {
hash: "ee".repeat(32),
offset: 0,
length: 4,
range_start: 0,
range_end: 1,
packed_start: 0,
packed_end: 4,
}],
};
let result = super::reconstruct_file_record_bytes(&store, &[], &record);
assert!(result.is_err());
}
#[test]
fn reconstruct_chunk_file_bytes_multiple_chunks_exact_capacity() {
let dir = tempfile::tempdir().unwrap();
let store = ServerObjectStore::local(dir.path()).unwrap();
let hash1 = "b1".repeat(32);
let hash2 = "b2".repeat(32);
let data1 = b"hello ";
let data2 = b"world";
let key1 = chunk_object_key(&hash1).unwrap();
let key2 = chunk_object_key(&hash2).unwrap();
let path1 = dir.path().join(key1.as_str());
let path2 = dir.path().join(key2.as_str());
std::fs::create_dir_all(path1.parent().unwrap()).unwrap();
std::fs::create_dir_all(path2.parent().unwrap()).unwrap();
std::fs::write(&path1, data1).unwrap();
std::fs::write(&path2, data2).unwrap();
let chunks = vec![
FileChunkRecord {
hash: hash1,
offset: 0,
length: data1.len() as u64,
range_start: 0,
range_end: 1,
packed_start: 0,
packed_end: data1.len() as u64,
},
FileChunkRecord {
hash: hash2,
offset: data1.len() as u64,
length: data2.len() as u64,
range_start: 0,
range_end: 1,
packed_start: 0,
packed_end: data2.len() as u64,
},
];
let total_capacity = data1.len() + data2.len();
let result = reconstruct_chunk_file_bytes(&store, &chunks, total_capacity);
assert!(result.is_ok());
assert_eq!(result.unwrap(), b"hello world");
}
#[test]
fn read_full_object_non_local_length_one_overflow_edge() {
let store = ServerObjectStore::blackhole();
let key = ObjectKey::parse("test/nonlocal-edge").unwrap();
let result = read_full_object(&store, &key, 1);
assert!(matches!(result, Err(ServerError::NotFound)));
}
}