use std::path::Path;
use std::sync::OnceLock;
use std::time::{Duration, SystemTime};
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use crate::model::{CheckReport, Finding, Severity};
use crate::reader::{Dataset, PlaneIndex};
pub const DEFAULT_WINDOW_S: u64 = 300;
pub const WINDOW_ENV: &str = "OPENREADOUT_LIVE_WINDOW";
const TAIL_SLACK: u64 = 1 << 20;
const MAX_TAIL: u64 = 64 << 20;
const MIN_TAIL: u64 = 16 << 20;
static WINDOW: OnceLock<Duration> = OnceLock::new();
pub fn set_window(d: Duration) {
let _ = WINDOW.set(d);
}
pub fn window() -> Duration {
*WINDOW.get_or_init(|| {
std::env::var(WINDOW_ENV)
.ok()
.and_then(|v| v.trim().parse::<f64>().ok())
.filter(|s| s.is_finite() && *s >= 0.0)
.map_or(
Duration::from_secs(DEFAULT_WINDOW_S),
Duration::from_secs_f64,
)
})
}
#[derive(Debug, Clone, Default)]
#[non_exhaustive]
pub struct WriteState {
pub missing: Vec<String>,
pub complete: Vec<(u32, PlaneIndex)>,
pub expected_planes: Option<u64>,
pub tail_bytes: u64,
pub unit_bytes: Option<u64>,
pub append_consistent: bool,
pub modified: Option<SystemTime>,
pub evidence: Vec<String>,
}
impl WriteState {
pub fn new() -> Self {
WriteState {
append_consistent: true,
..Default::default()
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
#[serde(rename_all = "snake_case")]
#[non_exhaustive]
pub enum AcquisitionState {
InProgress,
Interrupted,
}
impl AcquisitionState {
pub fn as_str(self) -> &'static str {
match self {
AcquisitionState::InProgress => "in_progress",
AcquisitionState::Interrupted => "interrupted",
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
pub struct Acquisition {
pub state: AcquisitionState,
pub complete_planes: u64,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub expected_planes: Option<u64>,
pub modified_ago_s: f64,
pub window_s: f64,
pub missing: Vec<String>,
pub tail_bytes: u64,
pub evidence: Vec<String>,
}
impl Acquisition {
pub fn in_progress(&self) -> bool {
self.state == AcquisitionState::InProgress
}
}
pub fn seconds_since(t: SystemTime) -> f64 {
match SystemTime::now().duration_since(t) {
Ok(d) => d.as_secs_f64(),
Err(e) => -e.duration().as_secs_f64(),
}
}
pub fn assess(path: &Path, ws: &WriteState) -> Option<Acquisition> {
assess_with(path, ws, window())
}
pub fn assess_with(path: &Path, ws: &WriteState, window: Duration) -> Option<Acquisition> {
if !ws.append_consistent {
return None;
}
let limit = ws.unit_bytes.map_or(MAX_TAIL, |u| {
u.saturating_mul(2).saturating_add(TAIL_SLACK).max(MIN_TAIL)
});
if ws.tail_bytes > limit {
return None;
}
let modified = ws
.modified
.or_else(|| std::fs::metadata(path).and_then(|m| m.modified()).ok())?;
let ago = seconds_since(modified);
let mut evidence = ws.evidence.clone();
let w = window.as_secs_f64();
let recent = w > 0.0 && ago <= w && ago >= -60.0;
let state = if recent {
evidence.push(format!(
"modified {ago:.1} s ago, within the {w:.0} s live window"
));
AcquisitionState::InProgress
} else {
evidence.push(format!(
"unchanged for {ago:.0} s, longer than the {w:.0} s live window"
));
AcquisitionState::Interrupted
};
Some(Acquisition {
state,
complete_planes: ws.complete.len() as u64,
expected_planes: ws.expected_planes,
modified_ago_s: (ago * 10.0).round() / 10.0,
window_s: w,
missing: ws.missing.clone(),
tail_bytes: ws.tail_bytes,
evidence,
})
}
pub fn assess_dataset(ds: &dyn Dataset, path: &Path) -> Option<Acquisition> {
ds.write_state().and_then(|ws| assess(path, &ws))
}
pub const IN_PROGRESS_CODE: &str = "acquisition_in_progress";
pub fn apply_to_check(report: &mut CheckReport, acq: &Acquisition) {
report.acquisition = Some(acq.clone());
if !acq.in_progress() {
return;
}
for f in &mut report.findings {
if f.severity == Severity::Error {
f.severity = Severity::Warning;
f.message = format!("in progress: {}", f.message);
}
}
report.ok = !report
.findings
.iter()
.any(|f| f.severity == Severity::Error);
let expected = acq
.expected_planes
.map_or_else(String::new, |e| format!(" of {e}"));
report.findings.insert(
0,
Finding::warning(
IN_PROGRESS_CODE,
format!(
"the file is still being written (modified {:.1} s ago; missing: {}): {}{} planes complete; `ok` covers the complete part only, run `check` again when the acquisition ends",
acq.modified_ago_s,
if acq.missing.is_empty() {
"nothing yet".into()
} else {
acq.missing.join(", ")
},
acq.complete_planes,
expected
),
),
);
}
pub fn annotate_error(path: &Path, err: crate::Error) -> crate::Error {
let crate::Error::Corrupt {
format,
detail,
offset,
} = err
else {
return err;
};
let w = window().as_secs_f64();
let ago = std::fs::metadata(path)
.and_then(|m| m.modified())
.ok()
.map(seconds_since);
match ago {
Some(a) if w > 0.0 && a <= w && a >= -60.0 => crate::Error::Corrupt {
format,
detail: format!(
"{detail} [modified {a:.1} s ago: the file may still be being written; this reader cannot read it until the acquisition ends (`openreadout watch` reports dataset_complete)]"
),
offset,
},
_ => crate::Error::Corrupt {
format,
detail,
offset,
},
}
}
pub fn complete_set(ws: &WriteState) -> std::collections::HashSet<(u32, PlaneIndex)> {
ws.complete.iter().copied().collect()
}
#[cfg(test)]
mod tests {
use super::*;
fn ws(tail: u64) -> WriteState {
let mut w = WriteState::new();
w.missing.push("chunk map".into());
w.complete.push((0, PlaneIndex::default()));
w.tail_bytes = tail;
w.unit_bytes = Some(4096);
w.modified = Some(SystemTime::now());
w
}
#[test]
fn recent_and_consistent_is_in_progress() {
let a = assess_with(Path::new("x"), &ws(100), Duration::from_secs(60)).unwrap();
assert_eq!(a.state, AcquisitionState::InProgress);
assert_eq!(a.complete_planes, 1);
}
#[test]
fn old_is_interrupted() {
let mut w = ws(100);
w.modified = Some(SystemTime::now() - Duration::from_secs(3600));
let a = assess_with(Path::new("x"), &w, Duration::from_secs(60)).unwrap();
assert_eq!(a.state, AcquisitionState::Interrupted);
}
#[test]
fn damage_or_huge_tail_is_not_an_acquisition() {
let mut w = ws(100);
w.append_consistent = false;
assert!(assess_with(Path::new("x"), &w, Duration::from_secs(60)).is_none());
let w = ws(1 << 30);
assert!(assess_with(Path::new("x"), &w, Duration::from_secs(60)).is_none());
}
#[test]
fn zero_window_disables() {
let a = assess_with(Path::new("x"), &ws(0), Duration::ZERO).unwrap();
assert_eq!(a.state, AcquisitionState::Interrupted);
}
#[test]
fn check_findings_downgraded_only_in_progress() {
let mut r = CheckReport::new("x", "nd2");
r.push(Finding::error("truncated", "chunk map missing"));
let a = assess_with(Path::new("x"), &ws(0), Duration::from_secs(60)).unwrap();
apply_to_check(&mut r, &a);
assert!(r.ok);
assert_eq!(r.findings[0].code, IN_PROGRESS_CODE);
assert!(r.findings[1].message.starts_with("in progress:"));
}
}