use core::{
future::{Future, ready},
sync::atomic::{AtomicU64, Ordering},
};
use std::{
borrow::Cow,
fs::{self, File, OpenOptions},
io::{self, Write},
path::{Path, PathBuf},
};
use smol_bytes::Utf8Bytes;
use super::{
AsyncBlobEngine, AsyncBlobStore, AsyncLocate, BlobEngine, BlobLocation, BlobStore, Connected,
Locate,
};
use crate::{BlobRecord, Id, Identifiable, Transformer, TypedId};
static PUT_NONCE: AtomicU64 = AtomicU64::new(0);
const BASE32: &[u8; 32] = b"abcdefghijklmnopqrstuvwxyz234567";
const RESERVED: [&str; 22] = [
"con", "prn", "aux", "nul", "com1", "com2", "com3", "com4", "com5", "com6", "com7", "com8",
"com9", "lpt1", "lpt2", "lpt3", "lpt4", "lpt5", "lpt6", "lpt7", "lpt8", "lpt9",
];
pub const MAX_SEGMENT_LEN: usize = 255;
pub const MAX_PATH_LEN: usize = 1023;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Strategy {
V1,
}
impl Strategy {
const fn of_store() -> Self {
Self::V1
}
fn render(self, segment: &str, out: &mut String) {
match self {
Self::V1 => render_segment(segment, out),
}
}
}
fn is_verbatim(segment: &str) -> bool {
!segment.is_empty()
&& !segment.starts_with('=')
&& segment
.bytes()
.all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'_' || b == b'-' || b == b'.')
&& segment != "."
&& segment != ".."
&& !RESERVED.contains(&segment)
}
fn render_segment(segment: &str, out: &mut String) {
if is_verbatim(segment) {
out.push_str(segment);
return;
}
out.push('=');
let bytes = segment.as_bytes();
for chunk in bytes.chunks(5) {
let mut buf = [0u8; 5];
buf[..chunk.len()].copy_from_slice(chunk);
let packed = u64::from_be_bytes([0, 0, 0, buf[0], buf[1], buf[2], buf[3], buf[4]]);
let symbols = chunk.len() * 8 / 5 + usize::from(chunk.len() * 8 % 5 != 0);
for i in 0..symbols {
let shift = 35 - i * 5;
out.push(BASE32[((packed >> shift) & 0x1f) as usize] as char);
}
}
}
#[derive(Debug, Default)]
pub struct Segments {
rendered: Vec<String>,
}
impl Segments {
#[inline]
pub const fn new() -> Self {
Self {
rendered: Vec::new(),
}
}
#[inline]
pub fn push<I: Segment>(&mut self, id: &I) {
let mut out = String::new();
id.render(&mut out);
self.rendered.push(out);
}
#[inline]
pub fn push_word(&mut self, word: &'static str) {
self.rendered.push(word.to_owned());
}
#[inline]
pub fn as_slice(&self) -> &[String] {
&self.rendered
}
}
pub trait KeySegments {
fn segments(&self, out: &mut Segments);
}
pub trait Segment: Id + private::Render {}
impl<I: Id + private::Render> Segment for I {}
mod private {
#[diagnostic::on_unimplemented(
message = "`{Self}` has no filesystem rendering, so it cannot become a path segment",
label = "no rendering",
note = "the renderable set IS the id vocabulary, so if `{Self}` is a MEASURED member its \
`Render` impl is missing from `store::fs` — add it beside the row \
`id::for_each_id_member!` admits it on. If `{Self}` DELEGATES, the delegation row below \
already renders it and what is missing is the delegate's own rendering. If `{Self}` is \
neither, it is not a key field type at all: see `ingraph::Id`"
)]
pub trait Render {
fn render(&self, out: &mut String);
}
}
use private::Render;
macro_rules! render_display {
($($ty:ty),* $(,)?) => {
$(impl Render for $ty {
#[inline]
fn render(&self, out: &mut String) {
out.push_str(&self.to_string());
}
})*
};
}
render_display!(i8, i16, i32, i64, i128, u8, u16, u32, u64, u128);
macro_rules! render_nonzero {
($($ty:ty),* $(,)?) => {
$(impl Render for $ty {
#[inline]
fn render(&self, out: &mut String) {
out.push_str(&self.get().to_string());
}
})*
};
}
render_nonzero!(
core::num::NonZeroI8,
core::num::NonZeroI16,
core::num::NonZeroI32,
core::num::NonZeroI64,
core::num::NonZeroI128,
core::num::NonZeroU8,
core::num::NonZeroU16,
core::num::NonZeroU32,
core::num::NonZeroU64,
core::num::NonZeroU128,
);
impl Render for uuid::Uuid {
#[inline]
fn render(&self, out: &mut String) {
let mut buf = [0u8; uuid::fmt::Hyphenated::LENGTH];
out.push_str(self.hyphenated().encode_lower(&mut buf));
}
}
macro_rules! render_text {
($($ty:ty),* $(,)?) => {
$(impl Render for $ty {
#[inline]
fn render(&self, out: &mut String) {
Strategy::of_store().render(self.as_ref(), out);
}
})*
};
}
render_text!(String, Utf8Bytes);
#[cfg(feature = "smol_str")]
render_text!(smol_str::SmolStr);
impl Render for core::net::Ipv4Addr {
#[inline]
fn render(&self, out: &mut String) {
let [a, b, c, d] = self.octets();
out.push_str(&format!("{a}.{b}.{c}.{d}"));
}
}
impl Render for core::net::Ipv6Addr {
#[inline]
fn render(&self, out: &mut String) {
self.octets().render(out);
}
}
impl Render for core::net::IpAddr {
#[inline]
fn render(&self, out: &mut String) {
match self {
Self::V4(ip) => ip.render(out),
Self::V6(ip) => ip.render(out),
}
}
}
impl Render for core::net::SocketAddrV4 {
#[inline]
fn render(&self, out: &mut String) {
self.ip().render(out);
out.push('_');
self.port().render(out);
}
}
impl Render for core::net::SocketAddrV6 {
#[inline]
fn render(&self, out: &mut String) {
self.ip().render(out);
out.push('_');
self.port().render(out);
out.push('_');
self.flowinfo().render(out);
out.push('_');
self.scope_id().render(out);
}
}
impl Render for core::net::SocketAddr {
#[inline]
fn render(&self, out: &mut String) {
match self {
Self::V4(addr) => addr.render(out),
Self::V6(addr) => addr.render(out),
}
}
}
impl<T> Render for T
where
T: crate::DelegatesId,
T::Inner: Render,
{
#[inline]
fn render(&self, out: &mut String) {
<T::Inner as Render>::render(self.delegate(), out);
}
}
impl<const N: usize> Render for [u8; N] {
#[inline]
fn render(&self, out: &mut String) {
for byte in self {
out.push(char::from_digit(u32::from(byte >> 4), 16).expect("a nibble is a hex digit"));
out.push(char::from_digit(u32::from(byte & 0x0f), 16).expect("a nibble is a hex digit"));
}
}
}
#[cfg(feature = "mongo")]
impl Render for bson::Uuid {
#[inline]
fn render(&self, out: &mut String) {
uuid::Uuid::from_bytes(self.bytes()).render(out);
}
}
#[cfg(feature = "mongo")]
impl Render for bson::oid::ObjectId {
#[inline]
fn render(&self, out: &mut String) {
self.bytes().render(out);
}
}
fn renders_as_a_segment<I: Segment + Render>() {}
macro_rules! audit_segment {
($($ty:ty),* $(,)?) => {
$(
const _: fn() = renders_as_a_segment::<$ty>;
)*
};
}
crate::id::for_each_id_member!(audit_segment);
const _: () = {
struct Witness;
impl Identifiable for Witness {
type Id = TypedId<u64, Witness>;
fn id_ref(&self) -> &Self::Id {
unreachable!("the audit takes a function's address and never calls it")
}
}
let _: [fn(); 7] = [
renders_as_a_segment::<std::sync::Arc<u64>>,
renders_as_a_segment::<std::rc::Rc<u64>>,
renders_as_a_segment::<Box<u64>>,
renders_as_a_segment::<[u8; 16]>,
renders_as_a_segment::<TypedId<u64, Witness>>,
renders_as_a_segment::<Box<std::sync::Arc<u64>>>,
renders_as_a_segment::<std::sync::Arc<TypedId<u64, Witness>>>,
];
};
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SegmentTooLong {
position: usize,
got: usize,
limit: usize,
}
impl SegmentTooLong {
#[inline]
pub(crate) const fn new(position: usize, got: usize, limit: usize) -> Self {
Self {
position,
got,
limit,
}
}
#[inline]
pub const fn position(&self) -> usize {
self.position
}
#[inline]
pub const fn got(&self) -> usize {
self.got
}
#[inline]
pub const fn limit(&self) -> usize {
self.limit
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct PathTooLong {
path: PathBuf,
got: usize,
limit: usize,
}
impl PathTooLong {
#[inline]
pub(crate) const fn new(path: PathBuf, got: usize, limit: usize) -> Self {
Self { path, got, limit }
}
#[inline]
pub fn path(&self) -> &Path {
&self.path
}
#[inline]
pub const fn got(&self) -> usize {
self.got
}
#[inline]
pub const fn limit(&self) -> usize {
self.limit
}
}
#[derive(Debug)]
#[non_exhaustive]
pub enum LocalFsBlobError {
Io(io::Error),
Missing(PathBuf),
InvalidSlot(Utf8Bytes),
EmptyKey,
InvalidSuffix(Utf8Bytes),
SegmentTooLong(SegmentTooLong),
PathTooLong(PathTooLong),
NonUtf8Address(PathBuf),
}
impl core::fmt::Display for LocalFsBlobError {
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
match self {
Self::Io(error) => error.fmt(f),
Self::Missing(path) => write!(f, "no blob stored at {path:?}"),
Self::InvalidSlot(slot) => write!(
f,
"{slot:?} is not a valid slot: it must be an identifier — `[A-Za-z_][A-Za-z0-9_]*`"
),
Self::EmptyKey => {
f.write_str("the key rendered no path segments — a keyed blob needs at least one key field")
}
Self::InvalidSuffix(suffix) => write!(
f,
"{suffix:?} cannot end a blob address: a suffix may not be empty, `.` or `..`, and may not \
carry a separator or a NUL — each would invent a directory level the key never declared"
),
Self::SegmentTooLong(error) => write!(
f,
"component {} of the blob address renders to {} bytes, past the {} one path component may \
take",
error.position(),
error.got(),
error.limit()
),
Self::PathTooLong(error) => write!(
f,
"the blob address {:?} is {} bytes, past the {} a portable path may take",
error.path(),
error.got(),
error.limit()
),
Self::NonUtf8Address(path) => write!(
f,
"the blob path {path:?} is not valid UTF-8, so it cannot render to a location's address"
),
}
}
}
impl core::error::Error for LocalFsBlobError {
fn source(&self) -> Option<&(dyn core::error::Error + 'static)> {
match self {
Self::Io(error) => Some(error),
_ => None,
}
}
}
impl From<io::Error> for LocalFsBlobError {
#[inline]
fn from(error: io::Error) -> Self {
Self::Io(error)
}
}
#[derive(Debug, Clone)]
pub struct LocalFsStore {
root: PathBuf,
}
impl LocalFsStore {
pub fn open(root: impl AsRef<Path>) -> Result<Self, LocalFsBlobError> {
let root = root.as_ref().to_path_buf();
fs::create_dir_all(&root)?;
Ok(Self { root })
}
#[inline]
pub fn root(&self) -> &Path {
&self.root
}
}
#[derive(Debug, Clone)]
pub struct LocalFsBlob {
slot: Utf8Bytes,
}
impl LocalFsBlob {
#[inline]
pub fn new(slot: impl Into<Utf8Bytes>) -> Self {
Self { slot: slot.into() }
}
#[inline]
pub fn slot(&self) -> &str {
&self.slot
}
pub fn address<K>(
&self,
root: &Path,
key: &K,
suffix: Option<&str>,
) -> Result<PathBuf, LocalFsBlobError>
where
K: KeySegments,
{
resolve_path(&self.slot, root, key, suffix)
}
pub fn fetch<K>(
&self,
root: &Path,
key: &K,
suffix: Option<&str>,
) -> Result<Option<Vec<u8>>, LocalFsBlobError>
where
K: KeySegments,
{
read_optional_blob(&self.slot, root, key, suffix)
}
}
impl BlobStore for LocalFsStore {
type Options = PathBuf;
type Error = LocalFsBlobError;
type Handle = LocalFsBlob;
#[inline]
fn open(options: Self::Options) -> Result<Self, Self::Error> {
LocalFsStore::open(options)
}
#[inline]
fn blob(&self, slot: impl Into<Utf8Bytes>) -> Self::Handle {
LocalFsBlob::new(slot)
}
}
impl AsyncBlobStore for LocalFsStore {
type Options = PathBuf;
type Error = LocalFsBlobError;
type Handle = LocalFsBlob;
fn open(options: Self::Options) -> impl Future<Output = Result<Self, Self::Error>> + Send {
ready(<Self as BlobStore>::open(options))
}
#[inline]
fn blob(&self, slot: impl Into<Utf8Bytes>) -> Self::Handle {
<Self as BlobStore>::blob(self, slot)
}
}
pub trait LocalFsBlobRecord: BlobRecord + Transformer<Key: KeySegments> {}
impl<T> LocalFsBlobRecord for T where T: BlobRecord + Transformer<Key: KeySegments> {}
fn resolve_path<K>(
slot: &str,
root: &Path,
key: &K,
suffix: Option<&str>,
) -> Result<PathBuf, LocalFsBlobError>
where
K: KeySegments,
{
if !crate::slot::is_identifier(slot.as_bytes()) {
return Err(LocalFsBlobError::InvalidSlot(Utf8Bytes::from(slot)));
}
if let Some(suffix) = suffix
&& (suffix.is_empty() || suffix == "." || suffix == ".." || suffix.contains(['/', '\\', '\0']))
{
return Err(LocalFsBlobError::InvalidSuffix(Utf8Bytes::from(suffix)));
}
let mut rendered = Segments::new();
key.segments(&mut rendered);
let (leaf, parents) = rendered
.as_slice()
.split_last()
.ok_or(LocalFsBlobError::EmptyKey)?;
let file_name = match suffix {
Some(ext) => Cow::Owned(format!("{leaf}.{ext}")),
None => Cow::Borrowed(leaf.as_str()),
};
let mut path = root.to_path_buf();
for (position, component) in core::iter::once(slot)
.chain(parents.iter().map(String::as_str))
.chain(core::iter::once(file_name.as_ref()))
.enumerate()
{
let got = component.len();
if got > MAX_SEGMENT_LEN {
return Err(LocalFsBlobError::SegmentTooLong(SegmentTooLong::new(
position,
got,
MAX_SEGMENT_LEN,
)));
}
path.push(component);
}
let got = path.as_os_str().len();
if got > MAX_PATH_LEN {
return Err(LocalFsBlobError::PathTooLong(PathTooLong::new(
path,
got,
MAX_PATH_LEN,
)));
}
Ok(path)
}
pub fn blob_address(reference: &Path) -> Result<String, LocalFsBlobError> {
reference
.to_str()
.map(str::to_owned)
.ok_or_else(|| LocalFsBlobError::NonUtf8Address(reference.to_path_buf()))
}
impl<R> BlobEngine<R> for LocalFsBlob
where
R: LocalFsBlobRecord,
R::Bytes: AsRef<[u8]>,
{
type Options = ();
type Error = LocalFsBlobError;
type Payload = Vec<u8>;
type Connection = LocalFsStore;
type ReadContext<'a> = ();
type WriteContext<'a> = ();
fn put(
&self,
ctx: Connected<'_, Self::Connection, Self::WriteContext<'_>>,
record: &R,
) -> Result<(), Self::Error> {
store_blob::<R>(&self.slot, ctx.connection().root(), record)
}
fn get(
&self,
ctx: Connected<'_, Self::Connection, Self::ReadContext<'_>>,
key: &R::Key,
) -> Result<Self::Payload, Self::Error>
where
R: Transformer,
{
read_blob(&self.slot, ctx.connection().root(), key, R::extension())
}
fn read(
&self,
ctx: Connected<'_, Self::Connection, Self::ReadContext<'_>>,
key: &R::Key,
suffix: Option<&str>,
) -> Result<Option<Self::Payload>, Self::Error>
where
R: Transformer,
{
read_optional_blob(&self.slot, ctx.connection().root(), key, suffix)
}
}
impl<R> AsyncBlobEngine<R> for LocalFsBlob
where
R: LocalFsBlobRecord,
R::Bytes: AsRef<[u8]>,
{
type Options = ();
type Error = LocalFsBlobError;
type Payload = Vec<u8>;
type Connection = LocalFsStore;
type ReadContext<'a> = ();
type WriteContext<'a> = ();
fn put(
&self,
ctx: Connected<'_, Self::Connection, Self::WriteContext<'_>>,
record: &R,
) -> impl Future<Output = Result<(), Self::Error>> + Send {
ready(<Self as BlobEngine<R>>::put(self, ctx, record))
}
fn get(
&self,
ctx: Connected<'_, Self::Connection, Self::ReadContext<'_>>,
key: &R::Key,
) -> impl Future<Output = Result<Self::Payload, Self::Error>> + Send
where
R: Transformer,
{
ready(<Self as BlobEngine<R>>::get(self, ctx, key))
}
fn read(
&self,
ctx: Connected<'_, Self::Connection, Self::ReadContext<'_>>,
key: &R::Key,
suffix: Option<&str>,
) -> impl Future<Output = Result<Option<Self::Payload>, Self::Error>> + Send
where
R: Transformer,
{
ready(<Self as BlobEngine<R>>::read(self, ctx, key, suffix))
}
}
impl<R> Locate<R> for LocalFsBlob
where
R: LocalFsBlobRecord,
R::Bytes: AsRef<[u8]>,
{
type Ref<'a> = PathBuf;
type Location = BlobLocation;
type LocateError = LocalFsBlobError;
fn locate<'a>(
&'a self,
ctx: Connected<'a, Self::Connection, Self::ReadContext<'a>>,
key: &'a R::Key,
suffix: Option<&str>,
) -> Result<Self::Ref<'a>, LocalFsBlobError>
where
R: Transformer,
{
resolve_path(&self.slot, ctx.connection().root(), key, suffix)
}
fn location<'a>(&'a self, reference: Self::Ref<'a>) -> Result<BlobLocation, LocalFsBlobError> {
blob_address(&reference).map(BlobLocation::new)
}
}
impl<R> AsyncLocate<R> for LocalFsBlob
where
R: LocalFsBlobRecord,
R::Bytes: AsRef<[u8]>,
{
type Ref<'a> = PathBuf;
type Location = BlobLocation;
type LocateError = LocalFsBlobError;
fn locate<'a>(
&'a self,
ctx: Connected<'a, Self::Connection, Self::ReadContext<'a>>,
key: &'a R::Key,
suffix: Option<&str>,
) -> Result<Self::Ref<'a>, LocalFsBlobError>
where
R: Transformer,
{
<Self as Locate<R>>::locate(self, ctx, key, suffix)
}
fn location<'a>(&'a self, reference: Self::Ref<'a>) -> Result<BlobLocation, LocalFsBlobError> {
<Self as Locate<R>>::location(self, reference)
}
}
fn store_blob<R>(slot: &str, root: &Path, record: &R) -> Result<(), LocalFsBlobError>
where
R: LocalFsBlobRecord,
R::Bytes: AsRef<[u8]>,
{
let path = resolve_path(slot, root, &record.key(), R::extension())?;
publish(&path, record.bytes().as_ref())?;
Ok(())
}
fn read_blob<K>(
slot: &str,
root: &Path,
key: &K,
suffix: Option<&str>,
) -> Result<Vec<u8>, LocalFsBlobError>
where
K: KeySegments,
{
let path = resolve_path(slot, root, key, suffix)?;
match fs::read(&path) {
Ok(bytes) => Ok(bytes),
Err(err) if err.kind() == io::ErrorKind::NotFound => Err(LocalFsBlobError::Missing(path)),
Err(err) => Err(err.into()),
}
}
fn read_optional_blob<K>(
slot: &str,
root: &Path,
key: &K,
suffix: Option<&str>,
) -> Result<Option<Vec<u8>>, LocalFsBlobError>
where
K: KeySegments,
{
match read_blob(slot, root, key, suffix) {
Ok(bytes) => Ok(Some(bytes)),
Err(LocalFsBlobError::Missing(_)) => Ok(None),
Err(err) => Err(err),
}
}
fn publish(final_path: &Path, bytes: &[u8]) -> io::Result<()> {
let parent = final_path
.parent()
.expect("a key path always has a parent directory");
fs::create_dir_all(parent)?;
let nonce = PUT_NONCE.fetch_add(1, Ordering::Relaxed);
let file_name = final_path
.file_name()
.expect("a key path always has a file name");
let mut tmp_name = file_name.to_os_string();
tmp_name.push(format!(".tmp.{}.{nonce}", std::process::id()));
let tmp_path = parent.join(tmp_name);
let file = OpenOptions::new()
.create_new(true)
.write(true)
.open(&tmp_path)?;
if let Err(err) = stage_and_rename(file, bytes, &tmp_path, final_path) {
let _ = fs::remove_file(&tmp_path);
return Err(err);
}
Ok(())
}
fn stage_and_rename(
mut file: File,
bytes: &[u8],
tmp_path: &Path,
final_path: &Path,
) -> io::Result<()> {
file.write_all(bytes)?;
drop(file);
fs::rename(tmp_path, final_path)
}
#[cfg(test)]
mod tests;