use super::*;
impl PageStore {
pub(super) fn immutable_key(&self, hash: &[u8; 32]) -> String {
use std::fmt::Write;
let mut key = String::with_capacity(self.prefix.len() + 195);
key.push_str(&self.prefix);
for identity in [&self.scope.namespace, &self.scope.incarnation, hash] {
key.push('/');
for byte in identity {
write!(&mut key, "{byte:02x}").expect("String formatting");
}
}
key
}
#[cfg(test)]
pub(super) fn key(&self, descriptor: Descriptor) -> String {
self.immutable_key(&descriptor.hash)
}
pub(super) fn validate_immutable(
&self,
raw: &[u8],
hash: &[u8; 32],
len: usize,
charge: &Arc<Charge>,
) -> Result<(), ObjectLogError> {
use sha2::{Digest, Sha256};
if !Arc::ptr_eq(&charge.budget, &self.budget)
|| charge.count < self.allocation as u64
|| !(72..=PAGE_BYTES).contains(&len)
|| raw.len() != len
|| raw[8..40] != self.scope.namespace
|| raw[40..72] != self.scope.incarnation
|| Sha256::digest(raw).as_slice() != hash
{
return Err(ObjectLogError::Sequencer(
"immutable page hash, scope, length or reservation differs".into(),
));
}
Ok(())
}
pub(super) async fn read_immutable(
&self,
hash: &[u8; 32],
len: usize,
charge: &Arc<Charge>,
) -> Result<crate::MetadataSnapshot, ObjectLogError> {
if !(72..=PAGE_BYTES).contains(&len)
|| !Arc::ptr_eq(&charge.budget, &self.budget)
|| charge.count < self.allocation as u64
{
return Err(ObjectLogError::InvalidConfig(
"immutable page length or reservation".into(),
));
}
let observed = self
.store
.read_metadata(
&self.immutable_key(hash),
self.metadata_limits(len),
self.owners(charge.clone() as AppendOwner),
)
.await?;
let raw = observed
.bytes()
.ok_or_else(|| ObjectLogError::MissingObject("native immutable page".into()))?;
self.validate_immutable(raw, hash, len, charge)?;
Ok(observed)
}
pub(super) async fn write_immutable(
&self,
raw: Vec<u8>,
hash: &[u8; 32],
charge: &Arc<Charge>,
) -> Result<(), ObjectLogError> {
if raw.capacity() > PAGE_BYTES {
return Err(ObjectLogError::BudgetExceeded(
"immutable encoded page capacity".into(),
));
}
self.validate_immutable(&raw, hash, raw.len(), charge)?;
let len = raw.len();
let key = self.immutable_key(hash);
let source = Bytes::from_owner(UploadedPage {
bytes: raw,
_charge: charge.clone(),
});
let absent = self.store.absent(&key, charge.clone() as AppendOwner);
if let MetadataSwap::Conflict(current) = self
.store
.swap_metadata(
&key,
absent,
source.clone(),
self.metadata_limits(len),
self.owners(charge.clone() as AppendOwner),
)
.await?
{
let existing = current
.bytes()
.ok_or_else(|| ObjectLogError::MissingObject("native immutable page".into()))?;
self.validate_immutable(existing, hash, len, charge)?;
if existing != source.as_ref() {
return Err(ObjectLogError::Sequencer(
"immutable page bytes differ".into(),
));
}
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::LocalBlobStore;
fn scope() -> Scope {
Scope {
namespace: [1; 32],
incarnation: [2; 32],
}
}
#[tokio::test]
async fn codec_neutral_pages_require_exact_scope_hash_length_and_credit() {
use sha2::{Digest, Sha256};
let root = tempfile::tempdir().unwrap();
let local = Arc::new(LocalBlobStore::new(root.path()));
let budget = ByteBudget::new(1 << 20).unwrap();
let pages = PageStore::new(
local.clone(),
"pages".into(),
scope(),
budget.clone(),
crate::MetadataLimits {
payload_bytes: PAGE_BYTES,
version_bytes: 56,
},
)
.unwrap();
let charge = budget.charge(pages.allocation()).unwrap();
let mut raw = vec![0; 76];
raw[..8].copy_from_slice(b"MAPTEST1");
raw[8..40].copy_from_slice(&scope().namespace);
raw[40..72].copy_from_slice(&scope().incarnation);
raw[72..].copy_from_slice(&[0, 255, 0, 1]);
let hash: [u8; 32] = Sha256::digest(&raw).into();
pages
.write_immutable(raw.clone(), &hash, &charge)
.await
.unwrap();
pages
.write_immutable(raw.clone(), &hash, &charge)
.await
.unwrap();
let observed = pages
.read_immutable(&hash, raw.len(), &charge)
.await
.unwrap();
assert_eq!(observed.bytes(), Some(raw.as_slice()));
assert!(
pages
.read_immutable(&hash, raw.len() - 1, &charge)
.await
.is_err()
);
assert!(
pages
.read_immutable(&hash, PAGE_BYTES + 1, &charge)
.await
.is_err()
);
let foreign = ByteBudget::new(1 << 20)
.unwrap()
.charge(pages.allocation())
.unwrap();
assert!(
pages
.read_immutable(&hash, raw.len(), &foreign)
.await
.is_err()
);
let short = budget.charge(pages.allocation() - 1).unwrap();
assert!(
pages
.read_immutable(&hash, raw.len(), &short)
.await
.is_err()
);
let mut wrong_scope = raw.clone();
wrong_scope[8] ^= 1;
let wrong_hash = Sha256::digest(&wrong_scope).into();
assert!(
pages
.write_immutable(wrong_scope, &wrong_hash, &charge)
.await
.is_err()
);
let mut wrong_content = raw.clone();
wrong_content[75] ^= 1;
assert!(
pages
.write_immutable(wrong_content, &hash, &charge)
.await
.is_err()
);
let mut oversized_capacity = Vec::with_capacity(PAGE_BYTES + 1);
oversized_capacity.extend_from_slice(&raw);
assert!(
pages
.write_immutable(oversized_capacity, &hash, &charge)
.await
.is_err()
);
assert_eq!(local.list("pages").await.unwrap().len(), 1);
drop(observed);
drop(short);
drop(charge);
assert_eq!(budget.available(), 1 << 20);
let charge = budget.charge(pages.allocation()).unwrap();
let mut corrupted = raw.clone();
corrupted[75] ^= 1;
local
.put(&pages.immutable_key(&hash), Bytes::from(corrupted))
.await
.unwrap();
assert!(
pages
.read_immutable(&hash, raw.len(), &charge)
.await
.is_err()
);
local
.put(&pages.immutable_key(&hash), Bytes::from_static(b"broken"))
.await
.unwrap();
assert!(
pages
.read_immutable(&hash, raw.len(), &charge)
.await
.is_err()
);
local.delete(&pages.immutable_key(&hash)).await.unwrap();
assert!(matches!(
pages.read_immutable(&hash, raw.len(), &charge).await,
Err(ObjectLogError::MissingObject(_))
));
drop(charge);
assert_eq!(budget.available(), 1 << 20);
}
#[tokio::test]
async fn create_only_writes_do_not_read_first_and_validate_what_they_find() {
use crate::native_v2::{
data::DataFrame,
operation::{ExecutionConfig, Resources},
semantic_map::TestCountingStore as CountingStore,
};
use sha2::{Digest, Sha256};
let root = tempfile::tempdir().unwrap();
let local = Arc::new(LocalBlobStore::new(root.path()));
let counting = CountingStore::new(local.clone());
let resources = Resources::new(ExecutionConfig {
workers: 2,
blocking_workers: 2,
operations: 1,
bytes: 1 << 20,
})
.unwrap();
let pages = PageStore::new(
counting.clone(),
"pages".into(),
scope(),
resources.bytes.clone(),
crate::MetadataLimits {
payload_bytes: PAGE_BYTES,
version_bytes: 56,
},
)
.unwrap();
resources
.run(move |physical| async move {
let pages = pages.in_operation(physical)?;
let charge = pages.charge_output(pages.allocation())?;
let page = |tail: [u8; 4]| {
let mut raw = vec![0; 76];
raw[..8].copy_from_slice(b"MAPTEST1");
raw[8..40].copy_from_slice(&scope().namespace);
raw[40..72].copy_from_slice(&scope().incarnation);
raw[72..].copy_from_slice(&tail);
let hash: [u8; 32] = Sha256::digest(&raw).into();
(raw, hash)
};
let (raw, hash) = page([0, 255, 0, 1]);
counting.take();
pages.write_immutable(raw.clone(), &hash, &charge).await?;
let calls = counting.take();
assert_eq!((calls.reads.len(), calls.writes.len()), (0, 1));
pages.write_immutable(raw.clone(), &hash, &charge).await?;
let calls = counting.take();
assert_eq!((calls.reads.len(), calls.writes.len()), (0, 1));
let (_, other_hash) = page([9, 9, 9, 9]);
let (other_raw, _) = page([9, 9, 9, 9]);
let key = pages.immutable_key(&other_hash);
let mut wrong = other_raw.clone();
wrong[75] ^= 1;
local.put(&key, Bytes::from(wrong)).await?;
assert!(
pages
.write_immutable(other_raw, &other_hash, &charge)
.await
.is_err()
);
let frame = DataFrame::encode(&pages, "logical/id", b"payload")?;
pages.upload_data("data/one", frame.bytes()).await?;
pages.upload_data("data/one", frame.bytes()).await?;
let calls = counting.take();
assert_eq!(calls.reads.len(), 0);
let same_identity = DataFrame::encode(&pages, "logical/id", b"different")?;
assert!(
pages
.upload_data("data/one", same_identity.bytes())
.await
.is_err()
);
let other_identity = DataFrame::encode(&pages, "another/id", b"payload")?;
assert!(
pages
.upload_data("data/one", other_identity.bytes())
.await
.is_err()
);
assert_eq!(local.get("data/one").await?.unwrap(), frame.bytes());
assert_eq!(counting.take().reads.len(), 0);
Ok(())
})
.await
.unwrap();
}
}