leviath-core 0.6.4

//! What a region entry holds: a list of typed parts, and the text they read as.
//!
//! An entry is any number of [`Part`]s, each with a mime type: a paragraph,
//! an image, a clip. Text parts keep their
//! bytes inline; every other part is a reference into the run's blob store.
//! The text those parts *read as* is kept beside them, rendered once when the
//! content is built: inline text as it is, and for each stored part the
//! stand-in its registry row produced at ingest. Everything that only knows
//! how to read text (the journal digest, a log line, a search index, an older
//! provider) reads that, through `Deref<Target = str>`, and never has to
//! learn what a part is.

use std::fmt;
use std::hash::{Hash, Hasher};
use std::ops::Deref;

use serde::{Deserialize, Deserializer, Serialize, Serializer};

use crate::mime::{Part, PartBody};

/// The typed content of one entry.
#[derive(Clone, PartialEq, Eq)]
pub struct EntryContent {
    parts: Vec<Part>,
    text: String,
}

impl EntryContent {
    /// One `text/plain` part.
    pub fn text(s: impl Into<String>) -> Self {
        Self::from_parts(vec![Part::text(s)])
    }

    /// Any parts, in order. The text rendering is built here, once.
    pub fn from_parts(parts: Vec<Part>) -> Self {
        let text = render(&parts);
        Self { parts, text }
    }

    /// The parts, in order.
    pub fn parts(&self) -> &[Part] {
        &self.parts
    }

    /// The parts, giving up the rendering.
    pub fn into_parts(self) -> Vec<Part> {
        self.parts
    }

    /// The text this content reads as.
    pub fn as_str(&self) -> &str {
        &self.text
    }

    /// The text this content reads as, owned.
    pub fn into_string(self) -> String {
        self.text
    }

    /// Whether every part is inline text.
    pub fn is_text_only(&self) -> bool {
        self.parts.iter().all(|p| !p.is_stored())
    }

    /// Whether any part is stored by reference.
    pub fn has_stored(&self) -> bool {
        !self.is_text_only()
    }

    /// The stored parts, in order.
    pub fn stored(&self) -> impl Iterator<Item = &Part> {
        self.parts.iter().filter(|p| p.is_stored())
    }

    /// How many parts are stored by reference.
    pub fn stored_count(&self) -> usize {
        self.stored().count()
    }

    /// The inline text alone, without the stand-ins the stored parts render
    /// as: what a caller that will rebuild the content around the same
    /// stored parts starts from.
    pub fn inline_text(&self) -> String {
        let mut out = String::new();
        for p in &self.parts {
            if let PartBody::Inline(s) = &p.body {
                if !out.is_empty() && !out.ends_with('\n') && !s.is_empty() {
                    out.push('\n');
                }
                out.push_str(s);
            }
        }
        out
    }

    /// The tokens this content costs a region: each part by [`Part::tokens`]
    /// when a registry is at hand, so a row's own token rule for a text type
    /// is honoured; by the byte heuristic without one, which is what the
    /// bundled text rows come to anyway.
    ///
    /// Either way a stored part is charged its stand-in, never its native
    /// estimate. In a region a stored blob is only a reference plus its short
    /// stand-in line, so that is what its budget pays; the native estimate
    /// (a per-byte rule over a multi-megabyte model is over a million tokens)
    /// is the cost of sending the bytes to a model that can read them, and is
    /// charged at request-build time, where it belongs.
    pub fn tokens(&self, registry: Option<&crate::mime::MimeRegistry>) -> usize {
        self.parts
            .iter()
            .map(|p| match (registry, &p.body) {
                (Some(registry), _) => p.tokens(registry),
                (None, PartBody::Inline(s)) => crate::text::estimate_tokens(s),
                (None, PartBody::Stored(b)) => crate::text::estimate_tokens(&b.stand_in),
            })
            .sum()
    }

    /// The same content with `part` appended.
    pub fn with_part(self, part: Part) -> Self {
        self.with_parts(std::iter::once(part))
    }

    /// The same content with every part of `parts` appended, rendered once.
    pub fn with_parts(mut self, parts: impl IntoIterator<Item = Part>) -> Self {
        self.parts.extend(parts);
        self.text = render(&self.parts);
        self
    }

    /// Whether this is exactly one unnamed `text/plain` part with no delivery
    /// override, which is what a bare string serialises as.
    fn is_plain_string(&self) -> bool {
        match self.parts.as_slice() {
            [p] => {
                p.name.is_none()
                    && p.deliver.is_none()
                    && p.mime_type.as_str() == "text/plain"
                    && matches!(p.body, PartBody::Inline(_))
            }
            _ => false,
        }
    }
}

/// Inline text as it is, a stored part as its stand-in, one per line.
fn render(parts: &[Part]) -> String {
    let mut out = String::new();
    for p in parts {
        let s = p.stand_in();
        if !out.is_empty() && !out.ends_with('\n') && !s.is_empty() {
            out.push('\n');
        }
        out.push_str(&s);
    }
    out
}

impl Default for EntryContent {
    fn default() -> Self {
        Self::text("")
    }
}

impl Deref for EntryContent {
    type Target = str;

    fn deref(&self) -> &str {
        &self.text
    }
}

impl AsRef<str> for EntryContent {
    fn as_ref(&self) -> &str {
        &self.text
    }
}

impl std::borrow::Borrow<str> for EntryContent {
    fn borrow(&self) -> &str {
        &self.text
    }
}

impl FromIterator<EntryContent> for String {
    fn from_iter<I: IntoIterator<Item = EntryContent>>(iter: I) -> Self {
        iter.into_iter().map(|c| c.text).collect()
    }
}

impl<'a> FromIterator<&'a EntryContent> for String {
    fn from_iter<I: IntoIterator<Item = &'a EntryContent>>(iter: I) -> Self {
        iter.into_iter().map(|c| c.text.as_str()).collect()
    }
}

impl fmt::Display for EntryContent {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        f.write_str(&self.text)
    }
}

impl fmt::Debug for EntryContent {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        if self.is_plain_string() {
            fmt::Debug::fmt(&self.text, f)
        } else {
            f.debug_struct("EntryContent")
                .field("parts", &self.parts)
                .field("text", &self.text)
                .finish()
        }
    }
}

impl Hash for EntryContent {
    fn hash<H: Hasher>(&self, state: &mut H) {
        self.text.hash(state);
        for p in &self.parts {
            p.mime_type.as_str().hash(state);
            p.name.hash(state);
            if let Some(b) = p.blob() {
                b.sha256.hash(state);
            }
        }
    }
}

impl From<String> for EntryContent {
    fn from(s: String) -> Self {
        Self::text(s)
    }
}

impl From<&str> for EntryContent {
    fn from(s: &str) -> Self {
        Self::text(s)
    }
}

impl From<Vec<Part>> for EntryContent {
    fn from(parts: Vec<Part>) -> Self {
        Self::from_parts(parts)
    }
}

impl From<EntryContent> for String {
    fn from(c: EntryContent) -> Self {
        c.text
    }
}

impl PartialEq<str> for EntryContent {
    fn eq(&self, other: &str) -> bool {
        self.text == other
    }
}

impl PartialEq<&str> for EntryContent {
    fn eq(&self, other: &&str) -> bool {
        self.text == *other
    }
}

impl PartialEq<String> for EntryContent {
    fn eq(&self, other: &String) -> bool {
        self.text == *other
    }
}

impl PartialEq<EntryContent> for str {
    fn eq(&self, other: &EntryContent) -> bool {
        self == other.text
    }
}

impl PartialEq<EntryContent> for &str {
    fn eq(&self, other: &EntryContent) -> bool {
        *self == other.text
    }
}

impl PartialEq<EntryContent> for String {
    fn eq(&self, other: &EntryContent) -> bool {
        *self == other.text
    }
}

/// The wire shape: a bare string for plain text, so a snapshot written
/// before parts existed reads back unchanged, and a list of parts otherwise.
#[derive(Serialize, Deserialize)]
#[serde(untagged)]
enum Wire {
    Text(String),
    Parts(Vec<Part>),
}

impl Serialize for EntryContent {
    fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
        if self.is_plain_string() {
            serializer.serialize_str(&self.text)
        } else {
            self.parts.serialize(serializer)
        }
    }
}

impl<'de> Deserialize<'de> for EntryContent {
    fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
        Ok(match Wire::deserialize(deserializer)? {
            Wire::Text(s) => Self::text(s),
            Wire::Parts(parts) => Self::from_parts(parts),
        })
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::mime::{Blob, MimeRegistry, MimeType};

    fn stored(name: &str) -> Part {
        let reg = MimeRegistry::builtin();
        let blob = Blob::new(MimeType::parse("image/png").unwrap(), vec![1, 2, 3]).named(name);
        Part::stored(blob.describe(&reg)).named(name)
    }

    #[test]
    fn text_reads_as_itself_and_serialises_as_a_string() {
        let c = EntryContent::text("hello");
        assert_eq!(c.as_str(), "hello");
        assert_eq!(&*c, "hello");
        assert_eq!(c.as_ref(), "hello");
        assert_eq!(c.to_string(), "hello");
        assert_eq!(format!("{c:?}"), "\"hello\"");
        assert!(c.is_text_only());
        assert!(!c.has_stored());
        assert_eq!(c.stored_count(), 0);
        assert_eq!(c.parts().len(), 1);
        assert_eq!(serde_json::to_string(&c).unwrap(), "\"hello\"");
        let back: EntryContent = serde_json::from_str("\"hello\"").unwrap();
        assert_eq!(back, c);
        assert_eq!(EntryContent::default().as_str(), "");
        assert_eq!(c.clone().into_string(), "hello");
        let s: String = c.clone().into();
        assert_eq!(s, "hello");
        assert_eq!(c.clone().into_parts().len(), 1);
    }

    #[test]
    fn equality_with_strings_in_both_directions() {
        let c = EntryContent::from("x");
        let owned = String::from("x");
        assert!(c == "x");
        assert!(c == *"x");
        assert!(c == owned);
        assert!("x" == c);
        assert!(*"x" == c);
        assert!(owned == c);
        let from_string: EntryContent = String::from("y").into();
        assert_ne!(from_string, c);
        let list = vec![EntryContent::text("a"), EntryContent::text("b")];
        assert_eq!(list.join("\n"), "a\nb");
        let joined: String = list.iter().collect();
        assert_eq!(joined, "ab");
        let owned: String = list.into_iter().collect();
        assert_eq!(owned, "ab");
    }

    #[test]
    fn stored_parts_render_their_stand_in_and_serialise_as_parts() {
        let c = EntryContent::from_parts(vec![Part::text("look:"), stored("a.png")]);
        assert_eq!(c.as_str(), "look:\n[image/png, 3 B] a.png");
        assert!(c.has_stored());
        assert_eq!(c.stored_count(), 1);
        assert!(format!("{c:?}").contains("EntryContent"));
        let json = serde_json::to_string(&c).unwrap();
        assert!(json.starts_with('['), "{json}");
        assert!(json.contains("\"sha256\""));
        let back: EntryContent = serde_json::from_str(&json).unwrap();
        assert_eq!(back, c);
        let with = EntryContent::text("first\n").with_part(stored("b.png"));
        assert_eq!(with.as_str(), "first\n[image/png, 3 B] b.png");
        let parts_only: EntryContent = vec![stored("c.png")].into();
        assert_eq!(parts_only.parts().len(), 1);
    }

    #[test]
    fn a_named_or_typed_text_part_is_not_a_bare_string() {
        let named = EntryContent::from_parts(vec![Part::text("t").named("n")]);
        assert!(serde_json::to_string(&named).unwrap().starts_with('['));
        let md = EntryContent::from_parts(vec![Part::inline(
            MimeType::parse("text/markdown").unwrap(),
            "# t",
        )]);
        assert!(serde_json::to_string(&md).unwrap().starts_with('['));
        let two = EntryContent::from_parts(vec![Part::text("a"), Part::text("b")]);
        assert_eq!(two.as_str(), "a\nb");
        assert!(serde_json::to_string(&two).unwrap().starts_with('['));
        let reg = MimeRegistry::builtin();
        let big_text = Blob::new(
            MimeType::parse("text/plain").unwrap(),
            b"stored text".to_vec(),
        );
        let stored_text = EntryContent::from_parts(vec![Part::stored(big_text.describe(&reg))]);
        assert!(
            serde_json::to_string(&stored_text)
                .unwrap()
                .starts_with('[')
        );
        assert!(serde_json::from_str::<EntryContent>("42").is_err());
        assert!(serde_json::from_str::<EntryContent>("{\"x\":1}").is_err());
        let empty = EntryContent::from_parts(vec![]);
        assert_eq!(empty.as_str(), "");
        assert!(serde_json::to_string(&empty).unwrap().starts_with('['));
    }

    #[test]
    fn a_region_with_accepts_refuses_other_types_and_says_what_it_takes() {
        use crate::region::{Region, RegionKind};
        let mut region = Region::new("art".into(), RegionKind::Pinned, 10_000);
        region.accepts = vec!["image/*".into(), "text/plain".into()];
        region
            .add_entry(
                EntryContent::from_parts(vec![Part::text("caption"), stored("a.png")]),
                5,
            )
            .unwrap();
        let err = region
            .add_entry(
                EntryContent::from_parts(vec![Part::inline(
                    MimeType::parse("text/markdown").unwrap(),
                    "# no",
                )]),
                5,
            )
            .unwrap_err();
        assert!(err.to_string().contains("image/*, text/plain"), "{err}");
        assert!(err.to_string().contains("text/markdown"), "{err}");
        assert!(region.accepts_content(&EntryContent::text("ok")).is_ok());
        let open = Region::new("any".into(), RegionKind::Pinned, 10);
        assert!(open.accepts_content(&EntryContent::text("x")).is_ok());
        assert_eq!(region.stored_count(), 1);
    }

    #[test]
    fn a_region_with_a_schema_takes_text_only() {
        use crate::region::schema::{ContentFormat, RegionSchema};
        use crate::region::{Region, RegionKind};
        let mut region = Region::new("plan".into(), RegionKind::Pinned, 10_000);
        region.schema = Some(RegionSchema::new(ContentFormat::Json));
        region.add_entry(EntryContent::text("{}"), 1).unwrap();
        let err = region
            .add_entry(EntryContent::from_parts(vec![stored("a.png")]), 5)
            .unwrap_err();
        assert!(
            err.to_string().contains("cannot hold a stored part"),
            "{err}"
        );
    }

    #[test]
    fn inline_text_and_token_hint_leave_the_stand_ins_out() {
        let c = EntryContent::from_parts(vec![
            Part::text("one"),
            stored("a.png"),
            Part::text(""),
            Part::text("two\n"),
            Part::text("three"),
        ]);
        assert_eq!(c.inline_text(), "one\ntwo\nthree");
        // A stored part costs its stand-in, not its native estimate, so a
        // large blob does not blow a region's budget.
        assert_eq!(
            c.tokens(None),
            crate::text::estimate_tokens("one")
                + crate::text::estimate_tokens(&stored("a.png").blob().unwrap().stand_in)
                + crate::text::estimate_tokens("two\n")
                + crate::text::estimate_tokens("three")
        );
        assert_eq!(EntryContent::text("").inline_text(), "");
    }

    /// With a registry the charge follows each type's own rule; with the
    /// bundled rows that is the byte heuristic, so the two answers agree, and
    /// a row of the operator's own moves it.
    #[test]
    fn a_registry_charges_inline_text_by_its_rule() {
        let c = EntryContent::from_parts(vec![Part::text("one two three"), stored("a.png")]);
        let builtin = MimeRegistry::builtin();
        assert_eq!(c.tokens(Some(&builtin)), c.tokens(None));
        let mut fixed = MimeRegistry::builtin();
        let table: toml::Table =
            toml::from_str("[\"text/plain\"]\ntokens = { fixed = 7 }\n").unwrap();
        fixed.layer(&table, "test").unwrap();
        let stand_in = crate::text::estimate_tokens(&stored("a.png").blob().unwrap().stand_in);
        assert_eq!(c.tokens(Some(&fixed)), 7 + stand_in);
    }

    #[test]
    fn a_large_stored_model_costs_its_stand_in_not_its_native_estimate() {
        // A per-byte rule over a multi-megabyte model yields a native estimate
        // in the millions; the region charge must be the short stand-in instead,
        // or a produced mesh is dropped from every region whose budget it
        // dwarfs (the routing path that store_routed takes).
        let stand_in = "[model/gltf-binary, 4.8 MB] model.glb";
        let part = Part::stored(crate::mime::BlobRef {
            sha256: "d".repeat(64),
            mime_type: MimeType::parse("model/gltf-binary").unwrap(),
            size: 5_000_000,
            width: None,
            height: None,
            duration_ms: None,
            tokens: 1_250_000,
            stand_in: stand_in.into(),
        });
        let c = EntryContent::from_parts(vec![part]);
        assert_eq!(c.tokens(None), crate::text::estimate_tokens(stand_in));
        assert!(
            c.tokens(None) < 100,
            "a mesh must fit an ordinary region budget"
        );
    }

    #[test]
    fn hashing_sees_parts_not_only_text() {
        use std::collections::hash_map::DefaultHasher;
        let h = |c: &EntryContent| {
            let mut s = DefaultHasher::new();
            c.hash(&mut s);
            s.finish()
        };
        let a = EntryContent::from_parts(vec![Part::text("x"), stored("a.png")]);
        let b = EntryContent::from_parts(vec![Part::text("x"), stored("a.png").named("a.png")]);
        assert_eq!(h(&a), h(&b));
        let c = EntryContent::text("x\n[image/png, 3 B] a.png");
        assert_ne!(h(&a), h(&c), "same text, different parts");
    }
}