corevm_host/

fs.rs

//! CoreVM FS that uses preimage store as a backend.
//!
//! Files are stored as a series of blocks. A file begins with a main block that stores metadata
//! and the actual first block. A directory is stored as a file with [`NodeKind::Dir`] specified in
//! the main block. Maximum file size is around 504 GiB.

mod error;
mod operations;
#[cfg(test)]
mod tests;

#[cfg(any(feature = "std", test))]
mod fs_std;

#[cfg(any(feature = "std", test))]
pub use self::fs_std::*;
pub use self::{error::*, operations::*};

use alloc::{borrow::Borrow, collections::VecDeque, ffi::CString, vec, vec::Vec};
use bytes::{Buf, Bytes};
use codec::{Compact, CompactLen, ConstEncodedLen, Decode, Encode, MaxEncodedLen};
use core::{ffi::CStr, ops::Deref};
use jam_types::{ServiceId, VecMap, VecSet, MAX_PREIMAGE_BLOB_LEN};

/// Minimum file block size in bytes.
///
/// Only the last file block can be smaller than that number.
///
/// Having min. size helps read metadata at the start of the file with a single lookup into the
/// preimage store.
pub const MIN_BLOCK_SIZE: usize = 64 * 1024;

/// Maximum file block size.
pub const MAX_BLOCK_SIZE: usize = MAX_PREIMAGE_BLOB_LEN;

/// Maximum file name length in bytes.
///
/// Uses the same value as Linux for compatibility. Includes the NUL byte.
pub const MAX_FILE_NAME_LEN: usize = 4096;

/// Single file block.
///
/// Up to [`MAX_BLOCK_SIZE`] bytes long.
#[derive(Clone, Debug)]
pub struct FileBlock(Bytes);

impl FileBlock {
	/// Create new file block from the provided data.
	///
	/// Fails if the data is larger than [`MAX_BLOCK_SIZE`].
	pub fn new(data: Bytes) -> Result<Self, InvalidBlock> {
		if data.len() > MAX_BLOCK_SIZE {
			return Err(InvalidBlock);
		}
		Ok(Self(data))
	}

	/// Convert into underlying `Bytes`.
	pub fn into_inner(self) -> Bytes {
		self.0
	}
}

impl Deref for FileBlock {
	type Target = [u8];
	fn deref(&self) -> &Self::Target {
		&self.0[..]
	}
}

impl AsRef<[u8]> for FileBlock {
	fn as_ref(&self) -> &[u8] {
		&self.0[..]
	}
}

/// File system node type.
#[derive(
	Encode, Decode, MaxEncodedLen, PartialEq, Eq, PartialOrd, Ord, Hash, Clone, Copy, Debug,
)]
pub enum NodeKind {
	/// File.
	File,
	/// Directory.
	Dir,
}

impl ConstEncodedLen for NodeKind {}

/// The main block that describes a file system node (either a file or a directory).
///
/// Contains the first block, the list of all other file blocks and the metadata.
#[derive(Debug)]
pub struct MainBlock {
	kind: NodeKind,
	file_size: u64,
	/// The exact size of every block except the first and the last one that can be smaller.
	block_size: u64,
	/// References of all file blocks except the first one.
	///
	/// The first block is a part of the superblock.
	block_refs: Vec<BlockRef>,
	/// The first file block.
	first_block: FileBlock,
}

impl MainBlock {
	/// Decode main block from the provided `Bytes`.
	pub fn decode(mut input: Bytes) -> Result<Self, InvalidBlock> {
		let mut slice = &input[..];
		let kind = NodeKind::decode(&mut slice).map_err(|_| InvalidBlock)?;
		let file_size = Compact::<u64>::decode(&mut slice).map_err(|_| InvalidBlock)?.0;
		let block_size = Compact::<u64>::decode(&mut slice).map_err(|_| InvalidBlock)?.0;
		let block_refs = Vec::<BlockRef>::decode(&mut slice).map_err(|_| InvalidBlock)?;
		let remaining_len = slice.len();
		input.advance(input.len() - remaining_len);
		let first_block = FileBlock::new(input)?;
		validate_main_block(file_size, block_size, &block_refs, &first_block)?;
		Ok(Self { kind, file_size, block_size, block_refs, first_block })
	}

	/// Encode main block into the provided `Vec`.
	pub fn encode_to(&self, output: &mut Vec<u8>) {
		self.kind.encode_to(output);
		Compact(self.file_size).encode_to(output);
		Compact(self.block_size).encode_to(output);
		self.block_refs.encode_to(output);
		output.extend_from_slice(self.first_block.as_ref());
	}

	/// Get node type.
	pub const fn kind(&self) -> NodeKind {
		self.kind
	}

	/// Get file size.
	pub const fn file_size(&self) -> u64 {
		self.file_size
	}

	/// Get block size.
	pub const fn block_size(&self) -> u64 {
		self.block_size
	}

	/// Get the first block.
	pub fn first_block(&self) -> &FileBlock {
		&self.first_block
	}

	/// Get the references to all blocks except the first one.
	pub fn block_refs(&self) -> &[BlockRef] {
		&self.block_refs
	}
}

fn validate_main_block(
	file_size: u64,
	block_size: u64,
	block_refs: &[BlockRef],
	first_block: &FileBlock,
) -> Result<(), InvalidBlock> {
	if !(MIN_BLOCK_SIZE as u64..=MAX_BLOCK_SIZE as u64).contains(&block_size) {
		log::trace!("Invalid block size: {block_size}");
		return Err(InvalidBlock);
	}
	let first_block_size = first_block.len();
	if first_block_size < MIN_BLOCK_SIZE && !block_refs.is_empty() {
		log::trace!(
			"Invalid first block size: first block size = {first_block_size}, \
            min. block size = {MIN_BLOCK_SIZE}, no. of hashes = {}",
			block_refs.len()
		);
		return Err(InvalidBlock);
	}
	let max = first_block_size as u64 + block_refs.len() as u64 * block_size;
	let min = if block_refs.is_empty() { first_block_size as u64 } else { max - block_size + 1 };
	if !(min..=max).contains(&file_size) {
		log::trace!("Invalid file size: {file_size} not in {min}..={max}");
		return Err(InvalidBlock);
	}
	let encoded_len = main_block_metadata_encoded_len(file_size, block_size, block_refs.len())
		.ok_or(InvalidBlock)? +
		first_block.len();
	if encoded_len > MAX_BLOCK_SIZE {
		log::trace!(
			"Encoded len is greater than the block size: {encoded_len} vs. {MAX_BLOCK_SIZE}"
		);
		return Err(InvalidBlock);
	}
	Ok(())
}

/// Includes everything except the first block.
fn main_block_metadata_encoded_len(
	file_size: u64,
	block_size: u64,
	num_blocks: usize,
) -> Option<usize> {
	(NodeKind::max_encoded_len() +
		Compact::<u64>::compact_len(&file_size) +
		Compact::<u64>::compact_len(&block_size))
	.checked_add(vec_encoded_len::<BlockRef>(num_blocks))
}

/// File name.
///
/// Internally this is a C-string to be compatible with UNIX system calls.
#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
pub struct FileName(CString);

impl FileName {
	/// Create new file name from the provide C-string.
	pub fn new(name: CString) -> Result<Self, InvalidPath> {
		validate_file_name(&name)?;
		Ok(Self(name))
	}

	/// Convert into underlying C string.
	pub fn into_inner(self) -> CString {
		self.0
	}
}

impl Encode for FileName {
	fn encode_to<O: codec::Output + ?Sized>(&self, output: &mut O) {
		self.0.to_bytes().encode_to(output)
	}
}

impl Decode for FileName {
	fn decode<I: codec::Input>(input: &mut I) -> Result<Self, codec::Error> {
		let mut bytes: Vec<u8> = Decode::decode(input)?;
		bytes.push(0_u8);
		let name = CString::from_vec_with_nul(bytes).map_err(|_| "Invalid C-string")?;
		validate_file_name(&name).map_err(|_| "Invalid file name")?;
		Ok(Self(name))
	}
}

impl MaxEncodedLen for FileName {
	fn max_encoded_len() -> usize {
		// -1 because we don't encode the NUL byte.
		MAX_FILE_NAME_LEN - 1 + Compact::<u32>(MAX_FILE_NAME_LEN as u32).encoded_size()
	}
}

impl AsRef<CStr> for FileName {
	fn as_ref(&self) -> &CStr {
		self.0.as_c_str()
	}
}

impl Deref for FileName {
	type Target = CStr;

	fn deref(&self) -> &Self::Target {
		self.0.as_c_str()
	}
}

impl Borrow<CStr> for FileName {
	fn borrow(&self) -> &CStr {
		self.0.as_c_str()
	}
}

impl Borrow<[u8]> for FileName {
	fn borrow(&self) -> &[u8] {
		// We borrow without the NUL byte to be able to resolve paths without unnecessary cloning.
		self.0.to_bytes()
	}
}

fn validate_file_name(name: &CStr) -> Result<(), InvalidPath> {
	let bytes = name.to_bytes_with_nul();
	if !(2..=MAX_FILE_NAME_LEN).contains(&bytes.len()) || bytes.contains(&b'/') {
		return Err(InvalidPath);
	}
	Ok(())
}

/// Unique file block identifier in the file system.
#[derive(
	Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Encode, Decode, MaxEncodedLen, Debug,
)]
pub struct BlockRef {
	/// Id of the service where all the file blocks are stored.
	pub service_id: ServiceId,
	/// Hash of the first file block.
	pub hash: Hash,
}

impl ConstEncodedLen for BlockRef {}

impl core::fmt::Display for BlockRef {
	fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
		write!(f, "{:x}:", self.service_id)?;
		for byte in self.hash.0.iter() {
			write!(f, "{byte:02x}")?;
		}
		Ok(())
	}
}

/// File block hash.
///
/// This is a type-safe wrapper around [`Hash`](jam_types::Hash) that is prefixed with
/// [`Hash::PREFIX`] when printed. Never equals the hash of the whole file (even if it fits into a
/// file block).
#[derive(
	Encode, Decode, MaxEncodedLen, PartialEq, Eq, PartialOrd, Ord, Hash, Clone, Copy, Debug, Default,
)]
pub struct Hash(pub jam_types::Hash);

impl Hash {
	/// Hash prefix when printed as string.
	pub const PREFIX: &str = "fs-";

	/// Computes the hash of the provided data.
	pub fn digest(data: &[u8]) -> Self {
		let h = blake2b_simd::Params::new().hash_length(32).hash(data);
		Self(h.as_bytes().try_into().expect("Hash length set to 32"))
	}

	/// Returns `true` if the hash is zero.
	pub fn is_zero(&self) -> bool {
		self.0.iter().all(|b| *b == 0)
	}
}

impl ConstEncodedLen for Hash {}

impl core::fmt::Display for Hash {
	fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
		f.write_str(Hash::PREFIX)?;
		for byte in self.0.iter() {
			write!(f, "{byte:02x}")?;
		}
		Ok(())
	}
}

// TODO @ivan `jamt` and `corevm-builder` need async versions of these traits

/// File block reader.
pub trait ReadBlock {
	/// Read file block referenced by `block_ref` from the content-addressable storage.
	fn read_block(&mut self, block_ref: &BlockRef) -> Result<Bytes, IoError>;
}

impl<R: ReadBlock + ?Sized> ReadBlock for &mut R {
	fn read_block(&mut self, block_ref: &BlockRef) -> Result<Bytes, IoError> {
		ReadBlock::read_block(*self, block_ref)
	}
}

impl ReadBlock for VecMap<BlockRef, Bytes> {
	fn read_block(&mut self, block_ref: &BlockRef) -> Result<Bytes, IoError> {
		Ok(self.get(block_ref).ok_or(IoError)?.clone())
	}
}

/// File block writer.
pub trait WriteBlock {
	/// Write file block stored in `buf` to the content-addressable storage.
	fn write_block(&mut self, service_id: ServiceId, buf: &[u8]) -> Result<(), IoError>;
}

impl<W: WriteBlock + ?Sized> WriteBlock for &mut W {
	fn write_block(&mut self, service_id: ServiceId, buf: &[u8]) -> Result<(), IoError> {
		WriteBlock::write_block(*self, service_id, buf)
	}
}

impl WriteBlock for VecMap<BlockRef, Bytes> {
	fn write_block(&mut self, service_id: ServiceId, buf: &[u8]) -> Result<(), IoError> {
		let block_ref = BlockRef { service_id, hash: Hash::digest(buf) };
		self.insert(block_ref, buf.to_vec().into());
		Ok(())
	}
}

/// Host file reader.
pub trait HostFileRead {
	/// Returns the remaining number of bytes in the input stream.
	fn remaining_len(&mut self) -> Result<u64, IoError>;

	/// Reads `buf.len()` bytes from the input stream into the provided buffer.
	fn read_exact(&mut self, buf: &mut [u8]) -> Result<(), IoError>;
}

/// Host file writer.
pub trait HostFileWrite {
	/// Fully write the provided buffer to the output stream.
	fn write_all(&mut self, buf: &[u8]) -> Result<(), IoError>;
}

/// Host directory reader.
pub trait HostDirRead<F: HostFileRead> {
	/// Unique identifier of the file/directory.
	///
	/// On UNIX this is `(device id, inode)`. Used to handle file system loops.
	type Id: core::hash::Hash + core::cmp::Eq + core::cmp::Ord;

	fn next_entry(&mut self) -> Option<Result<HostDirEntry, IoError>>;
	fn open_file(&mut self, name: &FileName) -> Result<F, IoError>;
	fn open_dir(&mut self, name: &FileName) -> Result<(Self, Option<Self::Id>), IoError>
	where
		Self: Sized;
}

/// Host directory entry.
#[derive(Debug)]
pub struct HostDirEntry {
	pub kind: NodeKind,
	pub file_name: FileName,
}

/// Host directory writer.
pub trait HostDirWrite {
	type FileWrite: HostFileWrite;

	fn create_file(&mut self, name: &FileName) -> Result<Self::FileWrite, IoError>;
	fn create_dir(&mut self, name: &FileName) -> Result<Self, IoError>
	where
		Self: Sized;
}

/// Host writer.
///
/// Can be turned into either file or directory writer.
pub trait HostWrite {
	type FileWrite: HostFileWrite;
	type DirWrite: HostDirWrite;

	fn into_file_writer(self) -> Result<Self::FileWrite, IoError>;
	fn into_dir_writer(self) -> Result<Self::DirWrite, IoError>;
}

fn vec_encoded_len<T: ConstEncodedLen>(len: usize) -> usize {
	T::max_encoded_len() * len + Compact::<u64>::compact_len(&(len as u64))
}