typst_library/foundations/

bytes.rs

use std::any::Any;
use std::fmt::{self, Debug, Formatter};
use std::hash::{Hash, Hasher};
use std::ops::{Add, AddAssign, Deref};
use std::str::Utf8Error;
use std::sync::Arc;

use ecow::{EcoString, eco_format};
use serde::{Serialize, Serializer};
use typst_syntax::{Lines, Source};
use typst_utils::LazyHash;

use crate::diag::{StrResult, bail};
use crate::foundations::{Array, Reflect, Repr, Str, Value, cast, func, scope, ty};

/// A sequence of bytes.
///
/// This is conceptually similar to an array of @int[integers] between `{0}` and
/// `{255}`, but represented much more efficiently. You can iterate over it
/// using a @reference:scripting:loops[for loop].
///
/// You can convert
/// - a @str[string] or an @array[array] of integers to bytes with the @bytes
///   constructor
/// - bytes to a string with the @str constructor, with UTF-8 encoding
/// - bytes to an array of integers with the @array constructor
///
/// When @read[reading] data from a file, you can decide whether to load it as a
/// string or as raw bytes.
///
/// ```example
/// #bytes((123, 160, 22, 0)) \
/// #bytes("Hello 😃")
///
/// #let data = read(
///   "rhino.png",
///   encoding: none,
/// )
///
/// // Magic bytes.
/// #array(data.slice(0, 4)) \
/// #str(data.slice(1, 4))
/// ```
#[ty(scope, cast)]
#[derive(Clone, Hash)]
pub struct Bytes(Arc<LazyHash<dyn Bytelike>>);

impl Bytes {
    /// Create `Bytes` from anything byte-like.
    ///
    /// The `data` type will directly back this bytes object. This means you can
    /// e.g. pass `&'static [u8]` or `[u8; 8]` and no extra vector will be
    /// allocated.
    ///
    /// If the type is `Vec<u8>` and the `Bytes` are unique (i.e. not cloned),
    /// the vector will be reused when mutating to the `Bytes`.
    ///
    /// If your source type is a string, prefer [`Bytes::from_string`] to
    /// directly use the UTF-8 encoded string data without any copying.
    pub fn new<T>(data: T) -> Self
    where
        T: AsRef<[u8]> + Send + Sync + 'static,
    {
        Self(Arc::new(LazyHash::new(data)))
    }

    /// Create `Bytes` from anything string-like, implicitly viewing the UTF-8
    /// representation.
    ///
    /// The `data` type will directly back this bytes object. This means you can
    /// e.g. pass `String` or `EcoString` without any copying.
    pub fn from_string<T>(data: T) -> Self
    where
        T: AsRef<str> + Send + Sync + 'static,
    {
        Self(Arc::new(LazyHash::new(StrWrapper(data))))
    }

    /// Return `true` if the length is 0.
    pub fn is_empty(&self) -> bool {
        self.as_slice().is_empty()
    }

    /// Return a view into the bytes.
    pub fn as_slice(&self) -> &[u8] {
        self
    }

    /// Try to view the bytes as an UTF-8 string.
    ///
    /// If these bytes were created via `Bytes::from_string`, UTF-8 validation
    /// is skipped.
    pub fn as_str(&self) -> Result<&str, Utf8Error> {
        self.inner().as_str()
    }

    /// Attempts to take ownership of an underlying vector. If this is not
    /// possible, returns a newly allocated vector with the byte data.
    ///
    /// For the underlying allocation to be reused, the bytes must have been
    /// created via [`Bytes::new`] from a [`Vec<u8>`] and the reference count
    /// must be 1.
    pub fn into_vec(mut self) -> Vec<u8> {
        match self.to_underlying_mut::<Vec<u8>>() {
            Some(vec) => std::mem::take(vec),
            None => self.as_slice().to_vec(),
        }
    }

    /// Attempts to take ownership of an underlying string or byte vector. If
    /// this is not possible, returns a newly allocated vector with the byte
    /// data.
    ///
    /// For the underlying allocation to be reused, the bytes must have been
    /// created via [`Bytes::new`] from a [`Vec<u8>`] or via
    /// [`Bytes::from_string`] from a [`String`] and the reference count must be
    /// 1.
    pub fn into_string(mut self) -> Result<String, IntoStringError> {
        if let Some(string) = self.to_underlying_string_mut::<String>() {
            return Ok(std::mem::take(string));
        }

        let result = if let Some(vec) = self.to_underlying_mut::<Vec<u8>>() {
            match String::from_utf8(std::mem::take(vec)) {
                Ok(string) => return Ok(string),
                Err(err) => {
                    let error = err.utf8_error();
                    *vec = err.into_bytes();
                    Err(error)
                }
            }
        } else {
            self.as_str().map(ToOwned::to_owned)
        };

        result.map_err(|error| IntoStringError { bytes: self, error })
    }

    /// Try to turn the bytes into a `Str`.
    ///
    /// - If these bytes were created via `Bytes::from_string::<Str>`, the
    ///   string is cloned directly.
    /// - If these bytes were created via `Bytes::from_string`, but from a
    ///   different type of string, UTF-8 validation is still skipped.
    pub fn to_str(&self) -> Result<Str, Utf8Error> {
        match (self.inner() as &dyn Any).downcast_ref::<Str>() {
            Some(string) => Ok(string.clone()),
            None => self.as_str().map(Into::into),
        }
    }

    /// Try to produce line metadata for these bytes. Fails if the bytes are not
    /// UTF-8 decodable.
    ///
    /// If the bytes were created from a [`Source`] file via
    /// [`Bytes::from_string`], the source file's line metadata is reused.
    /// Otherwise, line metadata is computed with internal memoization.
    pub fn lines(&self) -> Result<Lines<String>, Utf8Error> {
        #[comemo::memoize]
        fn compute(bytes: &Bytes) -> Result<Lines<String>, Utf8Error> {
            let text = bytes.as_str()?;
            Ok(Lines::new(text.to_string()))
        }

        // Small optimization: If this comes from a source file via
        // `Bytes::from_string`, we can directly use its lines.
        match self.to_underlying_string::<Source>() {
            Some(source) => Ok(source.lines().clone()),
            None => compute(self),
        }
    }
}

impl Bytes {
    /// Resolve an index or throw an out of bounds error.
    fn locate(&self, index: i64) -> StrResult<usize> {
        self.locate_opt(index).ok_or_else(|| out_of_bounds(index, self.len()))
    }

    /// Resolve an index, if it is within bounds.
    ///
    /// `index == len` is considered in bounds.
    fn locate_opt(&self, index: i64) -> Option<usize> {
        let len = self.as_slice().len();
        let wrapped =
            if index >= 0 { Some(index) } else { (len as i64).checked_add(index) };
        wrapped.and_then(|v| usize::try_from(v).ok()).filter(|&v| v <= len)
    }

    /// Try to access a vector this was built from via [`Bytes::new`].
    fn to_underlying_mut<T>(&mut self) -> Option<&mut T>
    where
        T: AsRef<[u8]> + Send + Sync + 'static,
    {
        Arc::get_mut(&mut self.0).and_then(|unique| {
            let inner: &mut dyn Bytelike = &mut **unique;
            (inner as &mut dyn Any).downcast_mut::<T>()
        })
    }

    /// Try to access a string this was built from via [`Bytes::from_string`].
    fn to_underlying_string<T>(&self) -> Option<&T>
    where
        T: AsRef<str> + Send + Sync + 'static,
    {
        (self.inner() as &dyn Any)
            .downcast_ref::<StrWrapper<T>>()
            .map(|wrapper| &wrapper.0)
    }

    /// Try to mutably access a string this was built from via [`Bytes::from_string`].
    fn to_underlying_string_mut<T>(&mut self) -> Option<&mut T>
    where
        T: AsRef<str> + Send + Sync + 'static,
    {
        Arc::get_mut(&mut self.0).and_then(|unique| {
            let inner: &mut dyn Bytelike = &mut **unique;
            (inner as &mut dyn Any)
                .downcast_mut::<StrWrapper<T>>()
                .map(|wrapper| &mut wrapper.0)
        })
    }

    /// Access the inner `dyn Bytelike`.
    fn inner(&self) -> &dyn Bytelike {
        &**self.0
    }
}

#[scope]
impl Bytes {
    /// Converts a value to bytes.
    ///
    /// - Strings are encoded in UTF-8.
    /// - Arrays of integers between `{0}` and `{255}` are converted directly.
    ///   The dedicated byte representation is much more efficient than the
    ///   array representation and thus typically used for large byte buffers
    ///   (e.g. image data).
    ///
    /// ```example
    /// #bytes("Hello 😃") \
    /// #bytes((123, 160, 22, 0))
    /// ```
    #[func(constructor)]
    pub fn construct(
        /// The value that should be converted to bytes.
        value: ToBytes,
    ) -> Bytes {
        value.0
    }

    /// The length in bytes.
    #[func(title = "Length")]
    pub fn len(&self) -> usize {
        self.as_slice().len()
    }

    /// Returns the byte at the specified index. Returns the default value if
    /// the index is out of bounds or fails with an error if no default value
    /// was specified.
    #[func]
    pub fn at(
        &self,
        /// The index at which to retrieve the byte.
        index: i64,
        /// A default value to return if the index is out of bounds.
        #[named]
        default: Option<Value>,
    ) -> StrResult<Value> {
        self.locate_opt(index)
            .and_then(|i| self.as_slice().get(i).map(|&b| Value::Int(b.into())))
            .or(default)
            .ok_or_else(|| out_of_bounds_no_default(index, self.len()))
    }

    /// Extracts a subslice of the bytes. Fails with an error if the start or
    /// end index is out of bounds.
    #[func]
    pub fn slice(
        &self,
        /// The start index (inclusive).
        start: i64,
        /// The end index (exclusive). If omitted, the whole slice until the end
        /// is extracted.
        #[default]
        end: Option<i64>,
        /// The number of items to extract. This is equivalent to passing
        /// `start + count` as the `end` position. Mutually exclusive with
        /// `end`.
        #[named]
        count: Option<i64>,
    ) -> StrResult<Bytes> {
        let start = self.locate(start)?;
        let end = end.or(count.map(|c| start as i64 + c));
        let end = self.locate(end.unwrap_or(self.len() as i64))?.max(start);
        let slice = &self.as_slice()[start..end];

        // We could hold a view into the original bytes here instead of
        // making a copy, but it's unclear when that's worth it. Java
        // originally did that for strings, but went back on it because a
        // very small view into a very large buffer would be a sort of
        // memory leak.
        Ok(Bytes::new(slice.to_vec()))
    }
}

impl Debug for Bytes {
    fn fmt(&self, f: &mut Formatter) -> fmt::Result {
        write!(f, "Bytes({})", self.len())
    }
}

impl Repr for Bytes {
    fn repr(&self) -> EcoString {
        eco_format!("bytes({})", self.len())
    }
}

impl Deref for Bytes {
    type Target = [u8];

    fn deref(&self) -> &Self::Target {
        self.inner().as_bytes()
    }
}

impl Eq for Bytes {}

impl PartialEq for Bytes {
    fn eq(&self, other: &Self) -> bool {
        self.0.eq(&other.0)
    }
}

impl AsRef<[u8]> for Bytes {
    fn as_ref(&self) -> &[u8] {
        self
    }
}

impl Add for Bytes {
    type Output = Self;

    fn add(mut self, rhs: Self) -> Self::Output {
        self += rhs;
        self
    }
}

impl AddAssign for Bytes {
    fn add_assign(&mut self, rhs: Self) {
        if rhs.is_empty() {
            // Nothing to do
        } else if self.is_empty() {
            *self = rhs;
        } else if let Some(vec) = self.to_underlying_mut::<Vec<u8>>() {
            vec.extend_from_slice(&rhs);
        } else {
            *self = Self::new([self.as_slice(), rhs.as_slice()].concat());
        }
    }
}

impl Serialize for Bytes {
    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
    where
        S: Serializer,
    {
        if serializer.is_human_readable() {
            serializer.serialize_str(&self.repr())
        } else {
            serializer.serialize_bytes(self)
        }
    }
}

/// An error that can occur in [`Bytes::into_string`].
#[derive(Debug)]
pub struct IntoStringError {
    pub bytes: Bytes,
    pub error: Utf8Error,
}

/// Any type that can back a byte buffer.
trait Bytelike: Any + Send + Sync {
    fn as_bytes(&self) -> &[u8];
    fn as_str(&self) -> Result<&str, Utf8Error>;
}

impl<T> Bytelike for T
where
    T: AsRef<[u8]> + Send + Sync + 'static,
{
    fn as_bytes(&self) -> &[u8] {
        self.as_ref()
    }

    fn as_str(&self) -> Result<&str, Utf8Error> {
        std::str::from_utf8(self.as_ref())
    }
}

impl Hash for dyn Bytelike {
    fn hash<H: Hasher>(&self, state: &mut H) {
        self.as_bytes().hash(state);
    }
}

/// Makes string-like objects usable with `Bytes`.
struct StrWrapper<T>(T);

impl<T> Bytelike for StrWrapper<T>
where
    T: AsRef<str> + Send + Sync + 'static,
{
    fn as_bytes(&self) -> &[u8] {
        self.0.as_ref().as_bytes()
    }

    fn as_str(&self) -> Result<&str, Utf8Error> {
        Ok(self.0.as_ref())
    }
}

/// A value that can be cast to bytes.
pub struct ToBytes(Bytes);

cast! {
    ToBytes,
    v: Str => Self(Bytes::from_string(v)),
    v: Array => Self(v.iter()
        .map(|item| match item {
            Value::Int(byte @ 0..=255) => Ok(*byte as u8),
            Value::Int(_) => bail!("number must be between 0 and 255"),
            value => Err(<u8 as Reflect>::error(value)),
        })
        .collect::<Result<Vec<u8>, _>>()
        .map(Bytes::new)?
    ),
    v: Bytes => Self(v),
}

/// The out of bounds access error message.
#[cold]
fn out_of_bounds(index: i64, len: usize) -> EcoString {
    eco_format!("byte index out of bounds (index: {index}, len: {len})")
}

/// The out of bounds access error message when no default value was given.
#[cold]
fn out_of_bounds_no_default(index: i64, len: usize) -> EcoString {
    eco_format!(
        "byte index out of bounds (index: {index}, len: {len}) \
         and no default value was specified",
    )
}

#[cfg(test)]
mod tests {
    use super::*;

    /// Round-tripping with lone ownership should retain the same string.
    #[test]
    fn test_bytes_into_string_lone() {
        let s1 = String::from("hello world");
        let p1 = s1.as_ptr();
        let s2 = Bytes::from_string(s1).into_string().unwrap();
        let p2 = s2.as_ptr();
        assert!(std::ptr::eq(p1, p2));
    }

    /// Round-tripping with shared ownership can yield a copy.
    #[test]
    fn test_bytes_into_string_shared() {
        let s1 = String::from("hello world");
        let p1 = s1.as_ptr();
        let x = Bytes::from_string(s1);
        let y = x.clone();
        let s2 = x.into_string().unwrap();
        let p2 = s2.as_ptr();
        let s3 = y.into_string().unwrap();
        let p3 = s3.as_ptr();
        // The first one yields a copy.
        assert!(!std::ptr::eq(p1, p2));
        // The last one yields the original string.
        assert!(std::ptr::eq(p1, p3));
    }

    /// Vector can also be reused as string.
    #[test]
    fn test_bytes_into_string_from_vec() {
        let v1 = String::from("hello world").into_bytes();
        let p1 = v1.as_ptr();
        let v2 = Bytes::new(v1).into_string().unwrap().into_bytes();
        let p2 = v2.as_ptr();
        assert!(std::ptr::eq(p1, p2));
    }

    /// UTF-8 error should retain the original bytes if it's a vector that could
    /// become a string.
    #[test]
    fn test_bytes_into_string_from_vec_error() {
        let s = b"hello world\xFF";
        let err = Bytes::new(Vec::from(s)).into_string().unwrap_err();
        assert_eq!(err.bytes.as_slice(), s);
    }
}