use crate::clock;
#[derive(Debug, PartialEq)]
pub enum Raw {
Mark { off: usize, when: f64 },
Count { off: usize, value: u8 },
Note { off: usize, text: String },
}
pub struct RawIter<'a> {
blob: &'a [u8],
i: usize,
}
pub fn raw(blob: &[u8]) -> RawIter<'_> {
RawIter { blob, i: 0 }
}
impl Iterator for RawIter<'_> {
type Item = Raw;
fn next(&mut self) -> Option<Raw> {
let n = self.blob.len();
while self.i < n {
let i = self.i;
let b = self.blob[i];
if b == 0x55 && i + 3 <= n && self.blob[i + 1] == 0xAA {
let kind = self.blob[i + 2];
if kind == 0x00 && i + 9 <= n {
let (yy, mm, dd) = (
self.blob[i + 3] as i32,
self.blob[i + 4] as i32,
self.blob[i + 5] as i32,
);
let (hh, mi, ss) = (
self.blob[i + 6] as i32,
self.blob[i + 7] as i32,
self.blob[i + 8] as i32,
);
let when = if (1..=12).contains(&mm)
&& (1..=31).contains(&dd)
&& hh <= 23
&& mi <= 59
&& ss <= 60
{
clock::from_parts(2000 + yy, mm, dd, hh, mi, ss)
} else {
None
};
self.i = i + 9;
if let Some(w) = when {
return Some(Raw::Mark { off: i, when: w });
}
continue;
}
if kind == 0x01 && i + 3 <= n {
self.i = i + 3;
continue;
}
if kind == 0x02 && i + 4 <= n {
let ln = self.blob[i + 3] as usize;
let end = (i + 4 + ln).min(n);
let text = String::from_utf8_lossy(&self.blob[i + 4..end])
.into_owned();
self.i = i + 4 + ln;
return Some(Raw::Note { off: i, text });
}
}
if b == 0xFF {
self.i = i + 1;
continue;
}
self.i = i + 1;
return Some(Raw::Count { off: i, value: b });
}
None
}
}
pub fn marks(blob: &[u8]) -> Vec<(usize, f64, usize)> {
let mut out: Vec<(usize, f64, usize)> = Vec::new();
let mut since = 0usize;
for r in raw(blob) {
match r {
Raw::Mark { off, when } => {
if let Some(last) = out.last_mut() {
last.2 = since;
}
out.push((off, when, 0));
since = 0;
}
Raw::Count { .. } => since += 1,
Raw::Note { .. } => {}
}
}
if let Some(last) = out.last_mut() {
last.2 = since;
}
out
}
pub fn sample_intervals(marks: &[(usize, f64, usize)]) -> Vec<Option<f64>> {
let mut raw_dts: Vec<Option<f64>> = vec![None; marks.len()];
for j in 0..marks.len().saturating_sub(1) {
let span = marks[j + 1].1 - marks[j].1;
let n = marks[j].2;
if n > 0 && span > 0.0 {
raw_dts[j] = Some(span / n as f64);
}
}
let mut good: Vec<f64> = raw_dts.iter().filter_map(|x| *x).collect();
if good.is_empty() {
return raw_dts;
}
good.sort_by(|a, b| a.partial_cmp(b).unwrap());
let median = good[good.len() / 2];
raw_dts
.into_iter()
.map(|x| match x {
Some(v) if 0.5 * median <= v && v <= 2.0 * median => Some(v),
_ => Some(median),
})
.collect()
}
pub struct Decoded {
pub off: usize,
pub when: Option<f64>,
pub dt: Option<f64>,
pub count: Option<u8>,
pub note: String,
}
pub fn records(blob: &[u8]) -> Vec<Decoded> {
let m = marks(blob);
let dts = sample_intervals(&m);
let at: std::collections::HashMap<usize, usize> =
m.iter().enumerate().map(|(j, x)| (x.0, j)).collect();
let mut out = Vec::new();
let mut base: Option<f64> = None;
let mut dt: Option<f64> = None;
let mut k = 0usize;
for r in raw(blob) {
match r {
Raw::Mark { off, when } => {
let j = at.get(&off).copied();
base = Some(when);
dt = j.and_then(|j| dts[j]);
k = 0;
out.push(Decoded { off, when: Some(when), dt, count: None,
note: String::new() });
}
Raw::Count { off, value } => match (base, dt) {
(Some(b), Some(d)) => {
out.push(Decoded { off, when: Some(b + k as f64 * d),
dt: Some(d), count: Some(value),
note: String::new() });
k += 1;
}
_ => out.push(Decoded { off, when: None, dt: None,
count: Some(value),
note: String::new() }),
},
Raw::Note { off, text } => {
out.push(Decoded { off, when: base, dt, count: None, note: text })
}
}
}
out
}
pub fn samples(blob: &[u8], offset: f64) -> Vec<(f64, u8, Option<f64>)> {
records(blob)
.into_iter()
.filter_map(|r| match (r.when, r.count) {
(Some(w), Some(c)) => Some((w + offset, c, r.dt)),
_ => None,
})
.collect()
}
pub fn rows_from_samples(
samples: &[(f64, u8, Option<f64>)],
spans: &[f64],
every: f64,
max_gap: f64,
src: &str,
site_for: &dyn Fn(f64) -> String,
) -> Vec<(f64, String)> {
use crate::analysis::Windows;
use crate::log;
let mut rows: Vec<(f64, String)> = Vec::new();
let mut w = Windows::new(spans);
let mut iv = log::Interval::new(spans.len());
let mut slot: Option<i64> = None;
let mut last: Option<f64> = None;
macro_rules! flush {
($w:expr, $iv:expr, $at:expr) => {
if $iv.seconds > 0.0 {
let averages: Vec<Option<f64>> =
spans.iter().map(|s| $w.average(*s)).collect();
let at: f64 = $at;
rows.push((
at,
log::row(at, $iv.cps(), $iv.counts, $iv.seconds, &averages,
&$iv.peaks, src, &site_for(at)),
));
}
$iv.reset();
};
}
for (when, count, dt) in samples {
if let Some(l) = last {
if when - l > max_gap {
flush!(w, iv, slot.unwrap_or(0) as f64 * every);
w = Windows::new(spans);
slot = None;
}
}
let here = log::slot_of(*when, every);
match slot {
None => slot = Some(here),
Some(s) if here != s => {
flush!(w, iv, s as f64 * every);
slot = Some(here);
}
_ => {}
}
w.add(*when, *count as u32);
let averages: Vec<Option<f64>> =
spans.iter().map(|s| w.average(*s)).collect();
iv.add(*count as u32, &averages, dt.unwrap_or(1.0));
last = Some(*when);
}
if let Some(s) = slot {
flush!(w, iv, s as f64 * every);
}
rows
}
pub struct Report {
pub samples: usize,
pub rows: usize,
pub added: usize,
pub clashed: usize,
pub first: Option<f64>,
pub last: Option<f64>,
pub holes: usize,
pub files: Vec<std::path::PathBuf>,
}
pub fn backfill(
blob: &[u8],
spans: &[f64],
every: f64,
max_gap: f64,
offset: f64,
dir: &std::path::Path,
serial: Option<&str>,
sites: &[(String, f64, String)],
one_file: Option<&std::path::Path>,
) -> Report {
use crate::log;
let mut samples = samples(blob, offset);
samples.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap());
let cut = crate::log::slot_of(crate::clock::now(), every) as f64 * every;
samples.retain(|s| s.0 < cut);
if samples.is_empty() {
return Report { samples: 0, rows: 0, added: 0, clashed: 0,
first: None, last: None, holes: 0, files: Vec::new() };
}
let serial_owned = serial.unwrap_or("").to_string();
let site_for = |when: f64| -> String {
if serial_owned.is_empty() {
String::new()
} else {
log::site_at(&serial_owned, when, sites).unwrap_or_default()
}
};
let rows = rows_from_samples(&samples, spans, every, max_gap,
log::SRC_FLASH, &site_for);
let head = log::header(spans);
let mut files: std::collections::BTreeMap<std::path::PathBuf,
Vec<(f64, String)>> =
std::collections::BTreeMap::new();
match one_file {
Some(p) => {
files.insert(p.to_path_buf(), rows.clone());
}
None => {
for r in &rows {
files.entry(log::path(r.0, dir, serial))
.or_default()
.push(r.clone());
}
}
}
let (mut added, mut clashed) = (0usize, 0usize);
for (path, group) in &files {
if let Ok((a, c)) = log::merge(path, &head, group, every) {
added += a;
clashed += c;
}
}
let holes = rows.windows(2).filter(|p| p[1].0 - p[0].0 > every).count();
Report {
samples: samples.len(),
rows: rows.len(),
added,
clashed,
first: Some(samples[0].0),
last: Some(samples[samples.len() - 1].0),
holes,
files: files.keys().cloned().collect(),
}
}
#[cfg(test)]
mod tests {
use super::*;
fn mark(y: u8, mo: u8, d: u8, h: u8, mi: u8, s: u8) -> Vec<u8> {
vec![0x55, 0xAA, 0x00, y, mo, d, h, mi, s]
}
fn tick() -> Vec<u8> {
vec![0x55, 0xAA, 0x01]
}
fn note(text: &str) -> Vec<u8> {
let mut v = vec![0x55, 0xAA, 0x02, text.len() as u8];
v.extend_from_slice(text.as_bytes());
v
}
fn image(parts: &[Vec<u8>]) -> Vec<u8> {
parts.concat()
}
#[test]
fn a_timestamp_is_nine_bytes_and_the_marker_after_it_is_three() {
let img = image(&[
mark(26, 9, 4, 12, 0, 0), tick(), vec![1, 2],
mark(26, 9, 4, 12, 0, 3), tick(), vec![3],
]);
let counts: Vec<u8> = raw(&img)
.filter_map(|r| match r {
Raw::Count { value, .. } => Some(value),
_ => None,
})
.collect();
assert_eq!(counts, vec![1, 2, 3], "no 170, no swallowed marker");
assert_eq!(marks(&img).len(), 2);
}
#[test]
fn unwritten_flash_is_absence_and_not_a_count_of_255() {
let img = image(&[
mark(26, 9, 4, 12, 0, 0), tick(), vec![1],
vec![0xFF; 16],
mark(26, 9, 4, 12, 0, 2), tick(), vec![2],
]);
let counts: Vec<u8> = raw(&img)
.filter_map(|r| match r {
Raw::Count { value, .. } => Some(value),
_ => None,
})
.collect();
assert_eq!(counts, vec![1, 2]);
}
#[test]
fn a_corrupt_timestamp_is_refused_rather_than_normalised() {
for bad in [(99u8, 99u8, 0u8, 0u8, 0u8), (0, 1, 0, 0, 0),
(1, 0, 0, 0, 0), (1, 1, 99, 0, 0), (1, 1, 0, 99, 0),
(1, 1, 0, 0, 99)] {
let img = image(&[
mark(26, bad.0, bad.1, bad.2, bad.3, bad.4),
tick(),
vec![1, 2],
]);
assert!(marks(&img).is_empty(), "accepted {:?}", bad);
}
}
#[test]
fn a_note_is_text_and_does_not_count_as_a_sample() {
let img = image(&[
mark(26, 9, 4, 12, 0, 0), tick(), vec![1],
note("bench"), vec![2],
]);
let notes: Vec<String> = raw(&img)
.filter_map(|r| match r {
Raw::Note { text, .. } => Some(text),
_ => None,
})
.collect();
assert_eq!(notes, vec!["bench".to_string()]);
assert_eq!(marks(&img)[0].2, 2, "the note is not one of the two counts");
}
#[test]
fn a_record_cut_off_by_the_end_of_a_tail_read_does_not_panic() {
let base = image(&[mark(26, 9, 4, 12, 0, 0), tick(), vec![1, 2, 3]]);
for tail in [vec![0x55], vec![0x55, 0xAA], vec![0x55, 0xAA, 0x00],
vec![0x55, 0xAA, 0x00, 26, 9], vec![0x55, 0xAA, 0x02],
vec![0x55, 0xAA, 0x02, 40, 0x61]] {
let mut img = base.clone();
img.extend_from_slice(&tail);
let _ = records(&img); }
}
#[test]
fn the_interval_is_measured_rather_than_assumed() {
let img = image(&[
mark(26, 9, 4, 12, 0, 0), tick(), vec![1; 9],
mark(26, 9, 4, 12, 0, 10), tick(), vec![1; 9],
mark(26, 9, 4, 12, 0, 20), tick(), vec![1; 3],
]);
let dts = sample_intervals(&marks(&img));
assert!((dts[0].unwrap() - 10.0 / 9.0).abs() < 1e-9, "{:?}", dts[0]);
}
#[test]
fn a_hole_is_a_hole_and_not_a_very_slow_stretch() {
let img = image(&[
mark(26, 9, 4, 12, 0, 0), tick(), vec![1; 9],
mark(26, 9, 4, 12, 0, 10), tick(), vec![1; 9],
mark(26, 9, 4, 12, 0, 20), tick(), vec![1; 9],
mark(26, 9, 4, 16, 0, 0), tick(), vec![1; 9],
mark(26, 9, 4, 16, 0, 10), tick(), vec![1; 3],
]);
let dts = sample_intervals(&marks(&img));
let median = 10.0 / 9.0;
assert!((dts[2].unwrap() - median).abs() < 1e-9,
"the four-hour stretch took the median, got {:?}", dts[2]);
}
#[test]
fn one_timestamp_measures_nothing_and_says_so() {
let img = image(&[mark(26, 9, 4, 12, 0, 0), tick(), vec![1, 2, 3]]);
assert_eq!(sample_intervals(&marks(&img)), vec![None]);
assert!(samples(&img, 0.0).is_empty());
}
#[test]
fn samples_before_the_first_timestamp_are_not_placed() {
let img = image(&[
vec![5, 6, 7],
mark(26, 9, 4, 12, 0, 0), tick(), vec![1, 2],
mark(26, 9, 4, 12, 0, 3), tick(), vec![0],
]);
let placed = samples(&img, 0.0);
assert!(placed.iter().all(|s| s.1 < 5),
"a partial read of the middle of the flash was placed anyway");
}
#[test]
fn the_offset_moves_every_sample_and_nothing_else() {
let img = image(&[
mark(26, 9, 4, 12, 0, 0), tick(), vec![1, 2],
mark(26, 9, 4, 12, 0, 3), tick(), vec![3],
]);
let a = samples(&img, 0.0);
let b = samples(&img, 1800.0);
assert_eq!(a.len(), b.len());
for (x, y) in a.iter().zip(b.iter()) {
assert!((y.0 - x.0 - 1800.0).abs() < 1e-9);
assert_eq!(x.1, y.1);
}
}
#[test]
fn an_empty_or_erased_image_decodes_to_nothing() {
assert!(records(&[]).is_empty());
assert!(records(&[0xFF; 64]).is_empty());
assert!(marks(&[0xFF; 64]).is_empty());
assert!(sample_intervals(&[]).is_empty());
}
#[test]
fn a_marker_byte_on_its_own_is_an_ordinary_count() {
let img = image(&[
mark(26, 9, 4, 12, 0, 0), tick(),
vec![0x55, 0x01, 0xAA],
mark(26, 9, 4, 12, 0, 3), tick(), vec![1],
]);
let counts: Vec<u8> = raw(&img)
.filter_map(|r| match r {
Raw::Count { value, .. } => Some(value),
_ => None,
})
.collect();
assert_eq!(counts, vec![0x55, 0x01, 0xAA, 1]);
}
}