use std::{collections::HashMap, fmt, hash::Hash, ops::Deref};
use serde::{
de::{DeserializeOwned, Error as _},
Deserialize, Deserializer,
};
use serde_json::value::RawValue;
use serde_with::DeserializeAs;
use tracing::warn;
pub struct SkipInvalidEntries;
impl<'de, T: DeserializeOwned> DeserializeAs<'de, Vec<T>> for SkipInvalidEntries {
fn deserialize_as<D: Deserializer<'de>>(deserializer: D) -> Result<Vec<T>, D::Error> {
let mut reader = EntryReader::new();
let entries = reader.read(Vec::<Box<RawValue>>::deserialize(deserializer)?, parse_entry);
reader
.finish()
.map_err(D::Error::custom)?;
Ok(entries)
}
}
impl<'de, K, T> DeserializeAs<'de, HashMap<K, Vec<T>>> for SkipInvalidEntries
where
K: Deserialize<'de> + Eq + Hash,
T: DeserializeOwned,
{
fn deserialize_as<D: Deserializer<'de>>(
deserializer: D,
) -> Result<HashMap<K, Vec<T>>, D::Error> {
let mut reader = EntryReader::new();
let groups = HashMap::<K, Vec<Box<RawValue>>>::deserialize(deserializer)?
.into_iter()
.map(|(key, raw)| (key, reader.read(raw, parse_entry)))
.collect();
reader
.finish()
.map_err(D::Error::custom)?;
Ok(groups)
}
}
fn parse_entry<T: DeserializeOwned>(raw: impl Deref<Target = RawValue>) -> serde_json::Result<T> {
serde_json::from_str(raw.get())
}
#[derive(Default)]
pub struct EntryReader {
kept: usize,
dropped: Vec<String>,
}
impl EntryReader {
pub fn new() -> Self {
EntryReader::default()
}
pub fn read<S, T, E: fmt::Display>(
&mut self,
raw: impl IntoIterator<Item = S>,
read_entry: impl Fn(S) -> Result<T, E>,
) -> Vec<T> {
let raw = raw.into_iter();
let mut entries = Vec::with_capacity(raw.size_hint().0);
for entry in raw {
match read_entry(entry) {
Ok(entry) => entries.push(entry),
Err(error) => self.dropped.push(error.to_string()),
}
}
self.kept += entries.len();
entries
}
pub fn finish(self) -> Result<(), String> {
let Some(first) = self.dropped.first() else { return Ok(()) };
let dropped = self.dropped.len();
if self.kept == 0 {
return Err(format!("all {dropped} entries failed to parse, the first with: {first}"));
}
warn!(
dropped,
kept = self.kept,
first_reason = first.as_str(),
"skipped entries the venue got wrong"
);
Ok(())
}
}
#[cfg(test)]
mod tests {
use std::sync::{Arc, Mutex};
use serde_with::serde_as;
use super::*;
#[serde_as]
#[derive(Debug, Deserialize)]
struct Entries(#[serde_as(deserialize_as = "SkipInvalidEntries")] Vec<u8>);
#[serde_as]
#[derive(Debug, Deserialize)]
struct Groups(#[serde_as(deserialize_as = "SkipInvalidEntries")] HashMap<String, Vec<u8>>);
#[test]
fn an_entry_the_venue_got_wrong_leaves_the_others_standing() {
let Entries(entries) = serde_json::from_str(r#"[1, "not a number", 3]"#).unwrap();
assert_eq!(entries, vec![1, 3]);
}
#[test]
fn an_empty_response_is_served_as_it_is() {
let Entries(entries) = serde_json::from_str("[]").unwrap();
assert_eq!(entries, Vec::<u8>::new());
}
#[test]
fn a_response_nothing_reads_in_is_an_error() {
let error = serde_json::from_str::<Entries>(r#"["not a number", "nor this"]"#).unwrap_err();
assert!(
error
.to_string()
.starts_with("all 2 entries failed to parse"),
"{error}"
);
}
#[test]
fn a_group_that_reads_as_nothing_does_not_cost_the_other_groups() {
let Groups(groups) =
serde_json::from_str(r#"{"mm1": ["not a number"], "mm2": [2, 3]}"#).unwrap();
assert_eq!(groups["mm1"], Vec::<u8>::new());
assert_eq!(groups["mm2"], vec![2, 3]);
}
#[test]
fn a_grouped_response_nothing_reads_in_is_an_error() {
let error =
serde_json::from_str::<Groups>(r#"{"mm1": ["not a number"], "mm2": ["nor this"]}"#)
.unwrap_err();
assert!(
error
.to_string()
.starts_with("all 2 entries failed to parse"),
"{error}"
);
}
#[test]
fn the_entries_that_were_dropped_are_warned_about() {
struct CaptureWriter(Arc<Mutex<Vec<u8>>>);
impl std::io::Write for CaptureWriter {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
self.0
.lock()
.unwrap()
.extend_from_slice(buf);
Ok(buf.len())
}
fn flush(&mut self) -> std::io::Result<()> {
Ok(())
}
}
let logs = Arc::new(Mutex::new(Vec::new()));
let writer = Arc::clone(&logs);
let subscriber = tracing_subscriber::fmt()
.with_env_filter(tracing_subscriber::EnvFilter::new("warn"))
.with_writer(move || CaptureWriter(Arc::clone(&writer)))
.with_ansi(false)
.finish();
let _guard = tracing::subscriber::set_default(subscriber);
let Entries(entries) =
serde_json::from_str(r#"[1, "not a number", "nor this either", 4]"#).unwrap();
assert_eq!(entries, vec![1, 4]);
let logs = String::from_utf8(logs.lock().unwrap().clone()).expect("logs are utf-8");
assert_eq!(logs.lines().count(), 1, "one warning for the response, got: {logs}");
assert!(
logs.contains("dropped=2") && logs.contains("kept=2"),
"the warning must count both halves, got: {logs}"
);
assert!(
logs.contains(r#"first_reason="invalid type: string \"not a number\""#),
"the warning must say why the first one went, got: {logs}"
);
}
#[test]
fn a_shape_that_is_no_longer_the_one_we_read_is_an_error() {
let error = serde_json::from_str::<Groups>(r#"{"mm1": [{"quantity": "1"}]}"#).unwrap_err();
assert!(
error
.to_string()
.starts_with("all 1 entries failed to parse"),
"{error}"
);
}
#[test]
fn a_group_that_is_not_a_run_of_entries_is_an_error() {
let error = serde_json::from_str::<Groups>(r#"{"mm1": "nothing to quote"}"#).unwrap_err();
assert_eq!(
error.to_string(),
"invalid type: string \"nothing to quote\", expected a sequence at line 1 column 26"
);
}
}