use crate::chunk::{FLAG_LAST_IN_GENERATION, HEADER_LEN, MAGIC, decode_header};
use std::io::{BufRead, BufReader, Read, Write};
use std::net::TcpStream;
use std::path::PathBuf;
use std::sync::mpsc;
use std::time::Instant;
#[derive(Debug, Clone)]
pub struct Snapshot {
pub seq: u32,
pub kind: String,
pub points: usize,
}
#[derive(Debug)]
pub enum Msg {
Received {
snap: Snapshot,
at: Instant,
},
Chunk {
seq: u32,
bytes: Vec<u8>,
last: bool,
at: Instant,
},
Delivered {
seq: u32,
delivery: String,
generation: u32,
bytes: u64,
fetch_start: Instant,
headers_at: Instant,
},
End,
Error(String),
}
pub fn host_port(url: &str) -> Result<String, String> {
let rest = url
.strip_prefix("http://")
.ok_or("server URL must start with http://")?;
Ok(rest.trim_end_matches('/').to_string())
}
type Headers = Vec<(String, String)>;
pub fn get(host: &str, path: &str) -> std::io::Result<(Headers, BufReader<TcpStream>)> {
let mut stream = TcpStream::connect(host)?;
stream.set_nodelay(true)?;
write!(
stream,
"GET {path} HTTP/1.1\r\nHost: {host}\r\nConnection: close\r\n\r\n"
)?;
stream.flush()?;
let mut r = BufReader::with_capacity(1 << 20, stream);
let mut line = String::new();
r.read_line(&mut line)?;
if line.split_whitespace().nth(1) != Some("200") {
return Err(std::io::Error::other(format!(
"GET {path}: {}",
line.trim_end()
)));
}
let mut headers = Vec::new();
loop {
line.clear();
if r.read_line(&mut line)? <= 2 {
break;
}
if let Some((k, v)) = line.split_once(':') {
headers.push((k.trim().to_ascii_lowercase(), v.trim().to_string()));
}
}
Ok((headers, r))
}
fn header<'a>(h: &'a [(String, String)], name: &str) -> Option<&'a str> {
h.iter().find(|(k, _)| k == name).map(|(_, v)| v.as_str())
}
pub fn parse_snapshot(json: &str) -> Option<Snapshot> {
let v: serde_json::Value = serde_json::from_str(json).ok()?;
Some(Snapshot {
seq: u32::try_from(v["seq"].as_u64()?).ok()?,
kind: v["kind"].as_str()?.to_string(),
points: usize::try_from(v["points"].as_u64()?).ok()?,
})
}
pub fn read_chunk(r: &mut impl Read) -> std::io::Result<Option<Vec<u8>>> {
let mut buf = vec![0u8; HEADER_LEN];
let mut got = 0;
while got < HEADER_LEN {
match r.read(&mut buf[got..])? {
0 if got == 0 => return Ok(None),
0 => return Err(std::io::ErrorKind::UnexpectedEof.into()),
n => got += n,
}
}
if buf[..4] != MAGIC {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
"not a chunk",
));
}
let n = u32::from_le_bytes([buf[16], buf[17], buf[18], buf[19]]) as usize;
let stride = usize::from(u16::from_le_bytes([buf[72], buf[73]]));
if !crate::chunk::STRIDES.contains(&stride) {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
format!("point stride {stride}"),
));
}
buf.resize(HEADER_LEN + n * stride, 0);
r.read_exact(&mut buf[HEADER_LEN..])?;
decode_header(&buf).map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
Ok(Some(buf))
}
fn fetch(
host: &str,
seq: u32,
have: Option<(u32, usize)>,
send: &impl Fn(Msg),
) -> std::io::Result<(String, u32, u64, Instant)> {
let path = match have {
Some((g, n)) => format!("/chunks/{seq}?have={g}.{n}"),
None => format!("/chunks/{seq}"),
};
let (headers, mut r) = get(host, &path)?;
let headers_at = Instant::now();
let delivery = header(&headers, "x-pb-delivery").unwrap_or("?").to_string();
let generation = header(&headers, "x-pb-generation")
.and_then(|g| g.parse().ok())
.unwrap_or(0);
let mut bytes = 0u64;
while let Some(c) = read_chunk(&mut r)? {
bytes += c.len() as u64;
let last = decode_header(&c).map(|h| h.flags & FLAG_LAST_IN_GENERATION != 0) == Ok(true);
send(Msg::Chunk {
seq,
bytes: c,
last,
at: Instant::now(),
});
if last {
break;
}
}
Ok((delivery, generation, bytes, headers_at))
}
enum Source {
Http,
Local(Local),
}
struct Local {
dir: PathBuf,
steps: Vec<crate::convert::Step>,
files: Vec<String>,
appends: Vec<bool>,
}
impl Local {
fn load(host: &str, dir: PathBuf) -> std::io::Result<Self> {
let (_, mut r) = get(host, "/manifest.json")?;
let mut text = String::new();
r.read_to_string(&mut text)?;
let bad =
|what: &str| std::io::Error::new(std::io::ErrorKind::InvalidData, what.to_string());
let v: serde_json::Value = serde_json::from_str(&text)
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
let events = v["events"]
.as_array()
.ok_or_else(|| bad("manifest without events"))?;
let mut local = Self {
dir,
steps: Vec::new(),
files: Vec::new(),
appends: Vec::new(),
};
for e in events {
let snap = parse_snapshot(&e.to_string()).ok_or_else(|| bad("bad manifest event"))?;
local.steps.push(crate::convert::Step {
seq: snap.seq,
preview: snap.kind == "preview",
points: snap.points,
});
local.files.push(
e["file"]
.as_str()
.ok_or_else(|| bad("event without file"))?
.to_string(),
);
local.appends.push(e["appends"].as_bool().unwrap_or(false));
}
Ok(local)
}
fn fetch(
&self,
seq: u32,
have: Option<(u32, usize)>,
send: &impl Fn(Msg),
) -> std::io::Result<(String, u32, u64, Instant)> {
use crate::convert::{Conversion, Plan, plan};
let idx = self
.steps
.iter()
.position(|s| s.seq == seq)
.ok_or_else(|| std::io::Error::other(format!("snapshot {seq} not in the manifest")))?;
let (generation, skip, delivery) = match plan(&self.steps, idx, have, |k| self.appends[k]) {
Plan::Full { generation } => (generation, 0, "full"),
Plan::Delta { generation, skip } => (generation, skip, "delta"),
};
let started = Instant::now();
let c = Conversion::start(self.dir.join(&self.files[idx]), skip, generation);
let n = c.len().map_err(std::io::Error::other)?;
let mut bytes = 0u64;
for i in 0..n {
let chunk = c.take(i).map_err(std::io::Error::other)?;
bytes += chunk.len() as u64;
send(Msg::Chunk {
seq,
bytes: chunk,
last: i + 1 == n,
at: Instant::now(),
});
}
Ok((delivery.to_string(), generation, bytes, started))
}
}
fn fetch_loop(host: &str, source: &Source, rx: mpsc::Receiver<Snapshot>, send: &impl Fn(Msg)) {
let mut have: Option<(u32, usize)> = None;
for snap in rx {
let fetch_start = Instant::now();
let got = match source {
Source::Http => fetch(host, snap.seq, have, send),
Source::Local(local) => local.fetch(snap.seq, have, send),
};
match got {
Ok((delivery, generation, bytes, headers_at)) => {
have = Some((generation, snap.points));
send(Msg::Delivered {
seq: snap.seq,
delivery,
generation,
bytes,
fetch_start,
headers_at,
});
}
Err(e) => {
send(Msg::Error(format!("snapshot {}: {e}", snap.seq)));
have = None; }
}
}
send(Msg::End);
}
pub fn spawn_replay(host: String, speed: f64, send: impl Fn(Msg) + Send + Sync + 'static) {
follow(host, None, speed, send);
}
pub fn spawn_replay_local(
host: String,
dir: PathBuf,
speed: f64,
send: impl Fn(Msg) + Send + Sync + 'static,
) {
follow(host, Some(dir), speed, send);
}
fn source(host: &str, dir: Option<PathBuf>) -> std::io::Result<Source> {
Ok(match dir {
Some(dir) => Source::Local(Local::load(host, dir)?),
None => Source::Http,
})
}
fn follow(
host: String,
dir: Option<PathBuf>,
speed: f64,
send: impl Fn(Msg) + Send + Sync + 'static,
) {
let send = std::sync::Arc::new(send);
let src = match source(&host, dir) {
Ok(src) => src,
Err(e) => {
send(Msg::Error(format!("manifest: {e}")));
send(Msg::End);
return;
}
};
let (tx, rx) = mpsc::channel::<Snapshot>();
let (h2, s2) = (host.clone(), send.clone());
std::thread::spawn(move || fetch_loop(&h2, &src, rx, &|m| s2(m)));
std::thread::spawn(move || {
let run = || -> std::io::Result<()> {
let (_, r) = get(&host, &format!("/events?speed={speed}"))?;
let mut event = String::new();
for line in r.lines() {
let line = line?;
let at = Instant::now();
if let Some(e) = line.strip_prefix("event: ") {
event = e.to_string();
} else if let Some(data) = line.strip_prefix("data: ") {
match event.as_str() {
"snapshot" => {
if let Some(snap) = parse_snapshot(data) {
send(Msg::Received {
snap: snap.clone(),
at,
});
let _ = tx.send(snap);
}
}
"end" => return Ok(()),
_ => {}
}
}
}
Ok(())
};
if let Err(e) = run() {
send(Msg::Error(format!("events: {e}")));
}
});
}
pub fn spawn_cold(host: String, send: impl Fn(Msg) + Send + Sync + 'static) {
cold(host, None, send);
}
pub fn spawn_cold_local(host: String, dir: PathBuf, send: impl Fn(Msg) + Send + Sync + 'static) {
cold(host, Some(dir), send);
}
fn cold(host: String, dir: Option<PathBuf>, send: impl Fn(Msg) + Send + Sync + 'static) {
std::thread::spawn(move || {
let run = || -> std::io::Result<()> {
let src = source(&host, dir.clone())?;
let (_, mut r) = get(&host, "/manifest.json")?;
let mut text = String::new();
r.read_to_string(&mut text)?;
let v: serde_json::Value = serde_json::from_str(&text)
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
let last = v["events"]
.as_array()
.and_then(|a| a.last())
.and_then(|e| parse_snapshot(&e.to_string()))
.ok_or_else(|| std::io::Error::other("empty manifest"))?;
send(Msg::Received {
snap: last.clone(),
at: Instant::now(),
});
let (tx, rx) = mpsc::channel();
tx.send(last).ok();
drop(tx);
fetch_loop(&host, &src, rx, &send);
Ok(())
};
if let Err(e) = run() {
send(Msg::Error(format!("cold: {e}")));
send(Msg::End);
}
});
}
#[derive(Debug, Clone)]
pub struct SlotEvent {
pub seq: u32,
pub kind: String,
pub points: u64,
pub bytes: u64,
}
#[derive(Debug)]
pub enum SlotMsg {
Received {
event: SlotEvent,
at: Instant,
},
Message {
seq: u32,
bytes: Vec<u8>,
state: bool,
fetch_start: Instant,
at: Instant,
},
End,
Error(String),
}
pub fn parse_slot_event(json: &str) -> Option<SlotEvent> {
let v: serde_json::Value = serde_json::from_str(json).ok()?;
Some(SlotEvent {
seq: u32::try_from(v["seq"].as_u64()?).ok()?,
kind: v["kind"].as_str()?.to_string(),
points: v["points"].as_u64()?,
bytes: v["bytes"].as_u64()?,
})
}
pub fn is_slot_stream(host: &str) -> std::io::Result<bool> {
let (_, mut r) = get(host, "/manifest.json")?;
let mut text = String::new();
r.read_to_string(&mut text)?;
let v: serde_json::Value = serde_json::from_str(&text)
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
Ok(v["stream"].as_str() == Some("slots"))
}
enum SlotSource {
Http,
Local(crate::slots::dir::StreamDir),
}
impl SlotSource {
fn new(dir: Option<PathBuf>) -> std::io::Result<Self> {
Ok(match dir {
Some(d) => Self::Local(crate::slots::dir::StreamDir::open(d)?),
None => Self::Http,
})
}
fn fetch(&mut self, host: &str, seq: u32, state: bool) -> std::io::Result<Vec<u8>> {
match self {
Self::Http => {
let path = if state {
format!("/slots/state?seq={seq}")
} else {
format!("/slots/{seq}")
};
let (headers, mut r) = get(host, &path)?;
let len = header(&headers, "content-length").and_then(|v| v.parse().ok());
let mut bytes = Vec::with_capacity(len.unwrap_or(0));
r.read_to_end(&mut bytes)?;
crate::slots::validate(&bytes)
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
Ok(bytes)
}
Self::Local(dir) => {
if dir.events().iter().all(|e| e.seq != seq) {
dir.refresh()?;
}
if !state {
return dir.message(seq);
}
let mut fold = crate::slots::Fold::new();
let seqs: Vec<u32> = dir
.events()
.iter()
.map(|e| e.seq)
.filter(|s| *s <= seq)
.collect();
for s in seqs {
fold.apply(&dir.message(s)?)
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
}
Ok(fold.message())
}
}
}
}
fn slot_fetch_loop(
host: &str,
mut source: SlotSource,
rx: mpsc::Receiver<SlotEvent>,
send: &impl Fn(SlotMsg),
) {
for event in rx {
let fetch_start = Instant::now();
match fetch_or_state(host, &mut source, event.seq) {
Ok((bytes, state)) => send(SlotMsg::Message {
seq: event.seq,
bytes,
state,
fetch_start,
at: Instant::now(),
}),
Err(e) => send(SlotMsg::Error(format!("event {}: {e}", event.seq))),
}
}
send(SlotMsg::End);
}
const SLOT_ATTEMPTS: u32 = 3;
const SLOT_RETRY: std::time::Duration = std::time::Duration::from_millis(100);
fn fetch_or_state(
host: &str,
source: &mut SlotSource,
seq: u32,
) -> std::io::Result<(Vec<u8>, bool)> {
let mut last = None;
for k in 0..SLOT_ATTEMPTS {
if k > 0 {
std::thread::sleep(SLOT_RETRY);
}
match source.fetch(host, seq, false) {
Ok(bytes) => return Ok((bytes, false)),
Err(e) => last = Some(e),
}
}
let e = last.expect("at least one attempt");
source
.fetch(host, seq, true)
.map(|b| (b, true))
.map_err(|s| std::io::Error::other(format!("{e}; folded state through {seq}: {s}")))
}
pub fn spawn_slots(
host: String,
dir: Option<PathBuf>,
speed: f64,
send: impl Fn(SlotMsg) + Send + Sync + 'static,
) {
let send = std::sync::Arc::new(send);
let source = match SlotSource::new(dir) {
Ok(s) => s,
Err(e) => {
send(SlotMsg::Error(format!("stream directory: {e}")));
send(SlotMsg::End);
return;
}
};
let (tx, rx) = mpsc::channel::<SlotEvent>();
let (h2, s2) = (host.clone(), send.clone());
std::thread::spawn(move || slot_fetch_loop(&h2, source, rx, &|m| s2(m)));
std::thread::spawn(move || {
let run = || -> std::io::Result<()> {
let (_, r) = get(&host, &format!("/events?speed={speed}"))?;
let mut event = String::new();
for line in r.lines() {
let line = line?;
let at = Instant::now();
if let Some(e) = line.strip_prefix("event: ") {
event = e.to_string();
} else if let Some(data) = line.strip_prefix("data: ") {
match event.as_str() {
"slots" => {
if let Some(ev) = parse_slot_event(data) {
send(SlotMsg::Received {
event: ev.clone(),
at,
});
let _ = tx.send(ev);
}
}
"end" => return Ok(()),
_ => {}
}
}
}
Ok(())
};
if let Err(e) = run() {
send(SlotMsg::Error(format!("events: {e}")));
}
});
}
pub fn spawn_slots_cold(
host: String,
dir: Option<PathBuf>,
send: impl Fn(SlotMsg) + Send + Sync + 'static,
) {
std::thread::spawn(move || {
let run = || -> std::io::Result<()> {
let mut source = SlotSource::new(dir)?;
let (_, mut r) = get(&host, "/manifest.json")?;
let mut text = String::new();
r.read_to_string(&mut text)?;
let v: serde_json::Value = serde_json::from_str(&text)
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
let last = v["events"]
.as_array()
.and_then(|a| a.last())
.and_then(|e| parse_slot_event(&e.to_string()))
.ok_or_else(|| std::io::Error::other("empty manifest"))?;
let seq = last.seq;
send(SlotMsg::Received {
event: last,
at: Instant::now(),
});
let fetch_start = Instant::now();
let bytes = source.fetch(&host, seq, true)?;
send(SlotMsg::Message {
seq,
bytes,
state: true,
fetch_start,
at: Instant::now(),
});
Ok(())
};
if let Err(e) = run() {
send(SlotMsg::Error(format!("cold: {e}")));
}
send(SlotMsg::End);
});
}
#[cfg(test)]
mod tests {
use super::*;
use crate::Point;
use crate::chunk::encode;
#[test]
fn slot_follower_retries_then_falls_back_to_the_folded_state() {
use crate::slots::{SlotHeader, encode};
use std::net::TcpListener;
let m = |seq: u32| {
let h = SlotHeader {
seq,
..Default::default()
};
encode(&h, &[]).unwrap()
};
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let host = listener.local_addr().unwrap().to_string();
let served = std::sync::Arc::new(std::sync::Mutex::new(Vec::<String>::new()));
let log = served.clone();
std::thread::spawn(move || {
let mut first_one = true;
for stream in listener.incoming() {
let mut stream = stream.unwrap();
let mut line = String::new();
let mut r = BufReader::new(&stream);
r.read_line(&mut line).unwrap();
let mut h = String::new();
while r.read_line(&mut h).unwrap() > 2 {
h.clear();
}
let path = line.split_whitespace().nth(1).unwrap_or("").to_string();
log.lock().unwrap().push(path.clone());
let body = match path.as_str() {
"/slots/1" if first_one => {
first_one = false;
None
}
"/slots/1" => Some(m(1)),
"/slots/state?seq=2" => Some(m(2)),
"/slots/3" => Some(m(3)),
_ => None,
};
let head = match &body {
Some(b) => format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\n\r\n", b.len()),
None => "HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\n\r\n".into(),
};
stream.write_all(head.as_bytes()).unwrap();
stream.write_all(&body.unwrap_or_default()).unwrap();
}
});
let (tx, rx) = mpsc::channel();
for seq in 1..=3 {
tx.send(SlotEvent {
seq,
kind: "refined".into(),
points: 0,
bytes: 0,
})
.unwrap();
}
drop(tx);
let got = std::sync::Mutex::new(Vec::new());
slot_fetch_loop(&host, SlotSource::Http, rx, &|msg| {
if let SlotMsg::Message { seq, state, .. } = msg {
got.lock().unwrap().push((seq, state));
}
});
assert_eq!(
got.into_inner().unwrap(),
vec![(1, false), (2, true), (3, false)]
);
let served = served.lock().unwrap();
assert_eq!(served.iter().filter(|p| *p == "/slots/1").count(), 2);
assert_eq!(
served.iter().filter(|p| *p == "/slots/2").count(),
SLOT_ATTEMPTS as usize
);
}
#[test]
fn parses_slot_events() {
let e = parse_slot_event(
r#"{"seq":6,"time_s":195.83,"t":195.83,"kind":"replace","zone":1,"points":1543927,"n_add":386295,"n_move":264502,"n_remove":395137,"bytes":14596488,"capacity":1939064,"url":"/slots/6"}"#,
)
.unwrap();
assert_eq!(
(e.seq, e.kind.as_str(), e.points, e.bytes),
(6, "replace", 1_543_927, 14_596_488)
);
assert!(parse_slot_event(r#"{"seq":1}"#).is_none());
}
#[test]
fn reads_back_to_back_chunks_until_the_end() {
let p = |x: f32| Point {
position: [x, 0.0, 0.0],
color: [1, 2, 3],
};
let a = encode(4, 0, 0, &[p(1.0), p(2.0)]);
let b = encode(4, 1, FLAG_LAST_IN_GENERATION, &[]);
let stream: Vec<u8> = [a.clone(), b.clone()].concat();
let mut r = &stream[..];
assert_eq!(read_chunk(&mut r).unwrap(), Some(a));
assert_eq!(read_chunk(&mut r).unwrap(), Some(b));
assert_eq!(read_chunk(&mut r).unwrap(), None);
let mut cut = &stream[..90];
assert!(read_chunk(&mut cut).is_err());
}
#[test]
fn parses_server_urls_and_snapshots() {
assert_eq!(
host_port("http://127.0.0.1:8700/").unwrap(),
"127.0.0.1:8700"
);
assert!(host_port("https://x").is_err());
let s = parse_snapshot(
r#"{"seq":3,"time_s":298.9,"kind":"preview","region":1,"file":"f","url":"/data/f","bytes":1,"points":371585}"#,
)
.unwrap();
assert_eq!((s.seq, s.kind.as_str(), s.points), (3, "preview", 371_585));
}
}