use serde::{Deserialize, Serialize};
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
#[non_exhaustive]
pub struct Metadata {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub title: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub description: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub og_image: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub keywords: Option<Vec<String>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub domain: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub pathname: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub resource_type: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub file_size: Option<u64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub extracted_data: Option<serde_json::Value>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub automation_data: Option<serde_json::Value>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub original_url: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub final_url: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub crawl_id: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub embedding: Option<Vec<f64>>,
#[serde(default, flatten)]
pub extra: serde_json::Map<String, serde_json::Value>,
}
impl Metadata {
pub fn is_empty(&self) -> bool {
*self == Metadata::default()
}
pub fn for_format(&self, format: &str) -> Option<Metadata> {
let block = self.extra.get(format)?;
if !block.is_object() {
return None;
}
serde_json::from_value(block.clone()).ok()
}
}
#[cfg(test)]
mod tests {
#![allow(
clippy::unwrap_used,
clippy::expect_used,
clippy::panic,
clippy::unreachable,
clippy::indexing_slicing,
clippy::string_slice
)]
use super::*;
#[test]
fn reads_a_sparse_block() {
let meta: Metadata =
serde_json::from_str(r#"{"title":"Example","og_image":null}"#).expect("metadata");
assert_eq!(meta.title.as_deref(), Some("Example"));
assert!(meta.og_image.is_none());
assert!(!meta.is_empty());
}
#[test]
fn reads_an_empty_block() {
let meta: Metadata = serde_json::from_str("{}").expect("metadata");
assert!(meta.is_empty());
}
#[test]
fn what_the_service_adds_is_kept_under_its_own_key() {
let meta: Metadata = serde_json::from_str(
r#"{"title":"Example","original_url":"https://example.com/a",
"final_url":null,"crawl_id":"3f1c","embedding":[0.25,0.5],
"yt_transcript":[{"text":"hello"}],"next_year":true}"#,
)
.expect("metadata");
assert_eq!(meta.original_url.as_deref(), Some("https://example.com/a"));
assert!(meta.final_url.is_none());
assert_eq!(meta.crawl_id.as_deref(), Some("3f1c"));
assert_eq!(meta.embedding.as_deref(), Some(&[0.25, 0.5][..]));
assert_eq!(meta.extra["yt_transcript"][0]["text"], "hello");
assert_eq!(meta.extra["next_year"], true);
assert!(meta.for_format("markdown").is_none());
}
#[test]
fn a_multi_format_block_keeps_each_format_apart() {
let meta: Metadata = serde_json::from_str(
r#"{"markdown":{"title":"As markdown","embedding":[0.1]},
"raw":{"title":"As html"},"yt_transcript":[]}"#,
)
.expect("metadata");
assert!(meta.title.is_none());
let markdown = meta.for_format("markdown").expect("markdown block");
assert_eq!(markdown.title.as_deref(), Some("As markdown"));
assert_eq!(markdown.embedding.as_deref(), Some(&[0.1][..]));
assert_eq!(
meta.for_format("raw").and_then(|m| m.title).as_deref(),
Some("As html")
);
assert!(meta.for_format("yt_transcript").is_none());
assert!(meta.for_format("text").is_none());
}
#[test]
fn unset_fields_are_not_written_back() {
let meta = Metadata {
title: Some("Example".into()),
..Metadata::default()
};
let json = serde_json::to_string(&meta).expect("serialize");
assert_eq!(json, r#"{"title":"Example"}"#);
}
}