use super::{SlotError, SlotHeader, add_capacity, encode};
use std::path::{Path, PathBuf};
#[derive(Debug, Clone, PartialEq)]
pub struct StreamEvent {
pub seq: u32,
pub t: f64,
pub zone: u32,
pub kind: String,
pub points: u64,
pub pack: Option<String>,
pub n_add: u32,
pub n_move: u32,
pub n_remove: u32,
pub bytes: u64,
}
#[derive(Debug, Clone, PartialEq)]
pub struct Manifest {
pub version: u32,
pub frame: String,
pub origin_gps: Option<[f64; 3]>,
pub capacity: u32,
pub events: Vec<StreamEvent>,
}
pub const MANIFEST_VERSION: u32 = 4;
pub fn parse_manifest(text: &str) -> Result<Manifest, String> {
let v: serde_json::Value = serde_json::from_str(text).map_err(|e| e.to_string())?;
let u32_of = |v: &serde_json::Value, k: &str| -> Result<u32, String> {
v[k].as_u64()
.and_then(|n| u32::try_from(n).ok())
.ok_or(format!("manifest: {k}"))
};
let version = u32_of(&v, "version")?;
if version != MANIFEST_VERSION {
return Err(format!(
"manifest version {version}, expected {MANIFEST_VERSION}"
));
}
let origin_gps = match v["origin_gps"].as_array() {
Some(a) if a.len() == 3 => {
let f = |i: usize| a[i].as_f64().ok_or("manifest: origin_gps");
Some([f(0)?, f(1)?, f(2)?])
}
_ => None,
};
let mut events = Vec::new();
for e in v["events"].as_array().ok_or("manifest: events")? {
events.push(StreamEvent {
seq: u32_of(e, "seq")?,
t: e["t"].as_f64().ok_or("manifest: t")?,
zone: u32_of(e, "zone")?,
kind: e["kind"].as_str().ok_or("manifest: kind")?.to_string(),
points: e["points"].as_u64().ok_or("manifest: points")?,
pack: e["pack"].as_str().map(str::to_string),
n_add: u32_of(e, "n_add")?,
n_move: u32_of(e, "n_move")?,
n_remove: u32_of(e, "n_remove")?,
bytes: e["bytes"].as_u64().ok_or("manifest: bytes")?,
});
}
events.sort_by_key(|e| e.seq);
Ok(Manifest {
version,
frame: v["frame"].as_str().unwrap_or("enu").to_string(),
origin_gps,
capacity: u32_of(&v, "capacity")?,
events,
})
}
fn invalid(e: impl std::fmt::Display) -> std::io::Error {
std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string())
}
#[derive(Debug)]
pub struct StreamDir {
dir: PathBuf,
manifest: Manifest,
caps: Vec<u32>,
}
impl StreamDir {
pub fn open(dir: impl Into<PathBuf>) -> std::io::Result<Self> {
let dir = dir.into();
let manifest = read_manifest(&dir)?;
Ok(Self {
dir,
manifest,
caps: Vec::new(),
})
}
pub fn path(&self) -> &Path {
&self.dir
}
pub fn manifest(&self) -> &Manifest {
&self.manifest
}
pub fn events(&self) -> &[StreamEvent] {
&self.manifest.events
}
pub fn refresh(&mut self) -> std::io::Result<Vec<StreamEvent>> {
let m = read_manifest(&self.dir)?;
let last = self.manifest.events.last().map_or(0, |e| e.seq);
let new = m.events.iter().filter(|e| e.seq > last).cloned().collect();
self.caps.truncate(m.events.len());
self.manifest = m;
Ok(new)
}
fn pack(&self, idx: usize) -> std::io::Result<Vec<u8>> {
let e = &self.manifest.events[idx];
let pack = match &e.pack {
Some(p) => std::fs::read(self.dir.join(p))?,
None => Vec::new(),
};
let want = super::pack_len(e.n_add, e.n_move, e.n_remove);
if pack.len() as u64 != want || pack.len() as u64 != e.bytes {
return Err(invalid(format!(
"event {}: pack is {} B, counts give {want} B, manifest says {} B",
e.seq,
pack.len(),
e.bytes
)));
}
Ok(pack)
}
fn capacity_before(&mut self, idx: usize) -> std::io::Result<u32> {
while self.caps.len() < idx {
let k = self.caps.len();
let e = &self.manifest.events[k];
let adds = match &e.pack {
Some(p) => {
use std::io::Read;
let mut buf = vec![0u8; e.n_add as usize * super::ADD_BYTES];
std::fs::File::open(self.dir.join(p))?.read_exact(&mut buf)?;
buf
}
None => Vec::new(),
};
let before = self.caps.last().copied().unwrap_or(0);
self.caps.push(before.max(add_capacity(&adds, e.n_add)));
}
Ok(if idx == 0 { 0 } else { self.caps[idx - 1] })
}
pub fn message(&mut self, seq: u32) -> std::io::Result<Vec<u8>> {
let idx = self
.manifest
.events
.iter()
.position(|e| e.seq == seq)
.ok_or_else(|| std::io::Error::other(format!("event {seq} not in the manifest")))?;
let before = self.capacity_before(idx)?;
let pack = self.pack(idx)?;
let e = &self.manifest.events[idx];
let capacity = before.max(add_capacity(&pack, e.n_add));
if self.caps.len() == idx {
self.caps.push(capacity);
}
let header = SlotHeader {
seq,
n_add: e.n_add,
n_move: e.n_move,
n_remove: e.n_remove,
capacity,
};
let msg = encode(&header, &pack).map_err(invalid)?;
super::validate(&msg).map_err(|e: SlotError| invalid(format!("event {seq}: {e}")))?;
Ok(msg)
}
pub fn messages(&mut self) -> std::io::Result<Vec<(StreamEvent, Vec<u8>)>> {
let events = self.manifest.events.clone();
events
.into_iter()
.map(|e| self.message(e.seq).map(|m| (e, m)))
.collect()
}
}
fn read_manifest(dir: &Path) -> std::io::Result<Manifest> {
let path = dir.join("manifest.json");
let text = std::fs::read_to_string(&path)?;
parse_manifest(&text).map_err(|e| invalid(format!("{}: {e}", path.display())))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::slots::{Diff, Fold, HEADER_LEN};
fn event_json(seq: u32, kind: &str, points: u64, pack: &[u8], counts: [u32; 3]) -> String {
let pack_field = if pack.is_empty() {
"null".to_string()
} else {
format!("\"packs/e_{seq:04}.bin\"")
};
format!(
r#"{{"seq":{seq},"t":{}.5,"zone":0,"kind":"{kind}","points":{points},"pack":{pack_field},"n_add":{},"n_move":{},"n_remove":{},"bytes":{}}}"#,
seq * 10,
counts[0],
counts[1],
counts[2],
pack.len()
)
}
fn write_stream(dir: &Path) {
use crate::slots::tests::msg;
std::fs::create_dir_all(dir.join("packs")).unwrap();
let events = [
(
"refined",
3,
msg(
1,
3,
&[(0, 1.0, [1; 3]), (1, 2.0, [2; 3]), (2, 3.0, [3; 3])],
&[],
&[],
),
[3, 0, 0],
),
(
"replace",
3,
msg(2, 4, &[(3, 4.0, [4; 3])], &[(1, 5.0, [5; 3])], &[0]),
[1, 1, 1],
),
(
"refined",
4,
msg(3, 4, &[(0, 6.0, [6; 3])], &[], &[]),
[1, 0, 0],
),
("final", 4, msg(4, 4, &[], &[], &[]), [0, 0, 0]),
];
let mut items = Vec::new();
for (k, (kind, points, m, counts)) in events.iter().enumerate() {
let seq = k as u32 + 1;
let pack = &m[HEADER_LEN..];
if !pack.is_empty() {
std::fs::write(dir.join(format!("packs/e_{seq:04}.bin")), pack).unwrap();
}
items.push(event_json(seq, kind, *points, pack, *counts));
}
let manifest = format!(
r#"{{"version":4,"frame":"enu","origin_gps":[36.5,127.25,29.9],"capacity":4,"events":[{}]}}"#,
items.join(",")
);
std::fs::write(dir.join("manifest.json"), manifest).unwrap();
}
#[test]
fn reads_a_stream_directory_in_seq_order() {
let dir = std::env::temp_dir().join(format!("pb-slots-dir-{}", std::process::id()));
std::fs::remove_dir_all(&dir).ok();
write_stream(&dir);
let mut s = StreamDir::open(&dir).unwrap();
let m = s.manifest().clone();
assert_eq!((m.version, m.frame.as_str(), m.capacity), (4, "enu", 4));
assert_eq!(m.origin_gps, Some([36.5, 127.25, 29.9]));
assert_eq!(m.events.len(), 4);
let third = s.message(3).unwrap();
assert_eq!(crate::slots::validate(&third).unwrap().capacity, 4);
let all = s.messages().unwrap();
let caps: Vec<u32> = all
.iter()
.map(|(_, m)| crate::slots::validate(m).unwrap().capacity)
.collect();
assert_eq!(caps, vec![3, 4, 4, 4]);
assert_eq!(all[3].1.len(), HEADER_LEN);
let mut fold = Fold::new();
let mut prev = 0i64;
for (e, msg) in &all {
let h = Diff::parse(msg).unwrap().header;
prev += i64::from(h.n_add) - i64::from(h.n_remove);
assert_eq!(prev, e.points as i64, "event {}", e.seq);
fold.apply(msg).unwrap();
}
assert_eq!(fold.points(), 4);
std::fs::write(dir.join("packs/e_0002.bin"), [0u8; 3]).unwrap();
assert!(s.message(2).is_err());
std::fs::remove_dir_all(&dir).ok();
}
#[test]
fn refresh_returns_only_new_events() {
let dir = std::env::temp_dir().join(format!("pb-slots-refresh-{}", std::process::id()));
std::fs::remove_dir_all(&dir).ok();
std::fs::create_dir_all(&dir).unwrap();
let one = format!(
r#"{{"version":4,"frame":"enu","origin_gps":null,"capacity":0,"events":[{}]}}"#,
event_json(1, "final", 0, &[], [0, 0, 0])
);
std::fs::write(dir.join("manifest.json"), one).unwrap();
let mut s = StreamDir::open(&dir).unwrap();
assert_eq!(s.manifest().origin_gps, None);
assert!(s.refresh().unwrap().is_empty());
write_stream(&dir);
let new: Vec<u32> = s.refresh().unwrap().iter().map(|e| e.seq).collect();
assert_eq!(new, vec![2, 3, 4]);
assert!(parse_manifest(r#"{"version":3,"capacity":0,"events":[]}"#).is_err());
std::fs::remove_dir_all(&dir).ok();
}
}