Skip to main content

openreadout_core/
live.rs

1//! Files that are still being written: the reader's structural evidence ([`WriteState`]) and
2//! the in-progress heuristic ([`assess`]) that turns it into an [`Acquisition`] status.
3//!
4//! A file an instrument is still writing lacks the structures its software writes last (a CZI
5//! subblock directory, the ND2 chunk map, OME-XML at the end of an OME-TIFF, Zarr chunks not
6//! yet on disk). Readers that know their format's write order report what is missing and which
7//! planes are already complete through [`crate::Dataset::write_state`]. [`assess`] then decides:
8//!
9//! - `in_progress`: the missing parts are only those written last, everything before them is
10//!   intact, the partial tail is no bigger than about one write unit, and the file (or, for a
11//!   directory store, its newest member) was modified within the live window
12//!   ([`window`], default [`DEFAULT_WINDOW_S`] seconds, `--live-window` / `OPENREADOUT_LIVE_WINDOW`).
13//! - `interrupted`: the same structural evidence, but the file has not changed for longer than
14//!   the window: an acquisition or copy that stopped. Readers report it as any truncated file
15//!   (truncation findings, exit 4 where they apply).
16//!
17//! Damage in the middle of a file (a bad segment id before the tail, a tail much larger than a
18//! write unit) is never in progress. Failure modes are documented in `book/src/guides/lab-shares.md`: a
19//! truncated copy made less than the window ago looks in progress, and a time-lapse whose
20//! interval is longer than the window looks interrupted between frames.
21
22use std::path::Path;
23use std::sync::OnceLock;
24use std::time::{Duration, SystemTime};
25
26use schemars::JsonSchema;
27use serde::{Deserialize, Serialize};
28
29use crate::model::{CheckReport, Finding, Severity};
30use crate::reader::{Dataset, PlaneIndex};
31
32/// Default live window in seconds: a file modified less than this long ago can be in progress.
33pub const DEFAULT_WINDOW_S: u64 = 300;
34
35/// Environment variable that sets the live window in seconds (`0` turns detection off).
36pub const WINDOW_ENV: &str = "OPENREADOUT_LIVE_WINDOW";
37
38/// Slack added to the size of one write unit when judging a partial tail (headers, padding).
39const TAIL_SLACK: u64 = 1 << 20;
40
41/// Largest partial tail accepted when the reader does not know its write unit.
42const MAX_TAIL: u64 = 64 << 20;
43
44/// A partial tail up to this size is always plausible (metadata blocks written at the end).
45const MIN_TAIL: u64 = 16 << 20;
46
47static WINDOW: OnceLock<Duration> = OnceLock::new();
48
49/// Set the live window for this process (first call wins; the CLI calls it from
50/// `--live-window`). Without a call, [`window`] reads [`WINDOW_ENV`] or uses the default.
51pub fn set_window(d: Duration) {
52    let _ = WINDOW.set(d);
53}
54
55/// The live window: how recently a file must have changed to be judged in progress.
56pub fn window() -> Duration {
57    *WINDOW.get_or_init(|| {
58        std::env::var(WINDOW_ENV)
59            .ok()
60            .and_then(|v| v.trim().parse::<f64>().ok())
61            .filter(|s| s.is_finite() && *s >= 0.0)
62            .map_or(
63                Duration::from_secs(DEFAULT_WINDOW_S),
64                Duration::from_secs_f64,
65            )
66    })
67}
68
69/// What a reader found about a file that is not finished: the parts its software writes last
70/// that are absent, and the planes already complete. Returned by
71/// [`crate::Dataset::write_state`] only for files that are incomplete.
72#[derive(Debug, Clone, Default)]
73#[non_exhaustive]
74pub struct WriteState {
75    /// Structures written at the end of an acquisition that are absent, in the format's own
76    /// words (`chunk map`, `subblock directory`, `OME-XML`, `chunk files`).
77    pub missing: Vec<String>,
78    /// Planes whose data is entirely on disk, in the order they were written.
79    pub complete: Vec<(u32, PlaneIndex)>,
80    /// Planes the file will hold when finished, when its metadata says so.
81    pub expected_planes: Option<u64>,
82    /// Bytes after the last complete unit: a plane, chunk or segment still being written.
83    pub tail_bytes: u64,
84    /// Size of one write unit (a frame chunk, a subblock segment, a page), for judging the tail.
85    pub unit_bytes: Option<u64>,
86    /// Everything before the tail is intact (no damage in the middle of the file), so the
87    /// incompleteness is explained by writing that has not finished.
88    pub append_consistent: bool,
89    /// Newest modification time among the files of the data set (directory stores); `None`
90    /// uses the path's own modification time.
91    pub modified: Option<SystemTime>,
92    /// The observations behind this state, in plain words.
93    pub evidence: Vec<String>,
94}
95
96impl WriteState {
97    /// An empty state that is append-consistent until a reader says otherwise.
98    pub fn new() -> Self {
99        WriteState {
100            append_consistent: true,
101            ..Default::default()
102        }
103    }
104}
105
106/// Whether an incomplete file is still being written.
107#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
108#[serde(rename_all = "snake_case")]
109#[non_exhaustive]
110pub enum AcquisitionState {
111    /// Recently modified and missing only what is written last: still acquiring.
112    InProgress,
113    /// Missing what is written last, but unchanged for longer than the live window: the
114    /// acquisition (or a copy) stopped before the end.
115    Interrupted,
116}
117
118impl AcquisitionState {
119    /// The snake_case name used in JSON.
120    pub fn as_str(self) -> &'static str {
121        match self {
122            AcquisitionState::InProgress => "in_progress",
123            AcquisitionState::Interrupted => "interrupted",
124        }
125    }
126}
127
128/// `acquisition` in `info`, `info --view structure`, `check` and `planes`: present only for
129/// files that are not finished. See `book/src/guides/lab-shares.md`.
130#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
131pub struct Acquisition {
132    /// `in_progress` or `interrupted`.
133    pub state: AcquisitionState,
134    /// Planes whose data is entirely on disk.
135    pub complete_planes: u64,
136    /// Planes the finished file will hold, when the metadata written so far says so.
137    #[serde(default, skip_serializing_if = "Option::is_none")]
138    pub expected_planes: Option<u64>,
139    /// Seconds since the file (or the newest member of a directory store) was modified.
140    pub modified_ago_s: f64,
141    /// The live window used for the decision, in seconds.
142    pub window_s: f64,
143    /// Structures written at the end of an acquisition that are absent.
144    pub missing: Vec<String>,
145    /// Bytes after the last complete unit (a unit still being written).
146    pub tail_bytes: u64,
147    /// The observations behind the decision.
148    pub evidence: Vec<String>,
149}
150
151impl Acquisition {
152    /// True when the file is still being written.
153    pub fn in_progress(&self) -> bool {
154        self.state == AcquisitionState::InProgress
155    }
156}
157
158/// Seconds since `t` (negative when `t` is in the future, e.g. clock skew on a share).
159pub fn seconds_since(t: SystemTime) -> f64 {
160    match SystemTime::now().duration_since(t) {
161        Ok(d) => d.as_secs_f64(),
162        Err(e) => -e.duration().as_secs_f64(),
163    }
164}
165
166/// Decide whether an incomplete file is in progress, with the process-wide [`window`].
167/// `None` when the evidence does not describe an unfinished write (damage in the middle, a
168/// tail too large for one write unit): the reader's own findings then stand.
169pub fn assess(path: &Path, ws: &WriteState) -> Option<Acquisition> {
170    assess_with(path, ws, window())
171}
172
173/// [`assess`] with an explicit window.
174pub fn assess_with(path: &Path, ws: &WriteState, window: Duration) -> Option<Acquisition> {
175    if !ws.append_consistent {
176        return None;
177    }
178    let limit = ws.unit_bytes.map_or(MAX_TAIL, |u| {
179        u.saturating_mul(2).saturating_add(TAIL_SLACK).max(MIN_TAIL)
180    });
181    if ws.tail_bytes > limit {
182        return None;
183    }
184    let modified = ws
185        .modified
186        .or_else(|| std::fs::metadata(path).and_then(|m| m.modified()).ok())?;
187    let ago = seconds_since(modified);
188    let mut evidence = ws.evidence.clone();
189    let w = window.as_secs_f64();
190    // Up to a minute in the future is clock skew between the instrument PC and this machine.
191    let recent = w > 0.0 && ago <= w && ago >= -60.0;
192    let state = if recent {
193        evidence.push(format!(
194            "modified {ago:.1} s ago, within the {w:.0} s live window"
195        ));
196        AcquisitionState::InProgress
197    } else {
198        evidence.push(format!(
199            "unchanged for {ago:.0} s, longer than the {w:.0} s live window"
200        ));
201        AcquisitionState::Interrupted
202    };
203    Some(Acquisition {
204        state,
205        complete_planes: ws.complete.len() as u64,
206        expected_planes: ws.expected_planes,
207        modified_ago_s: (ago * 10.0).round() / 10.0,
208        window_s: w,
209        missing: ws.missing.clone(),
210        tail_bytes: ws.tail_bytes,
211        evidence,
212    })
213}
214
215/// [`assess`] on an opened data set: `None` for finished files and formats that do not report
216/// a [`WriteState`].
217pub fn assess_dataset(ds: &dyn Dataset, path: &Path) -> Option<Acquisition> {
218    ds.write_state().and_then(|ws| assess(path, &ws))
219}
220
221/// Code of the `check` finding that marks a file still being written.
222pub const IN_PROGRESS_CODE: &str = "acquisition_in_progress";
223
224/// Adjust a `check` report for a file that is still being written: the findings about the
225/// unfinished tail become warnings (prefixed `in progress:`), an `acquisition_in_progress`
226/// finding explains why, and `ok` then covers the complete part only. Interrupted files keep
227/// their errors.
228pub fn apply_to_check(report: &mut CheckReport, acq: &Acquisition) {
229    report.acquisition = Some(acq.clone());
230    if !acq.in_progress() {
231        return;
232    }
233    for f in &mut report.findings {
234        if f.severity == Severity::Error {
235            f.severity = Severity::Warning;
236            f.message = format!("in progress: {}", f.message);
237        }
238    }
239    report.ok = !report
240        .findings
241        .iter()
242        .any(|f| f.severity == Severity::Error);
243    let expected = acq
244        .expected_planes
245        .map_or_else(String::new, |e| format!(" of {e}"));
246    report.findings.insert(
247        0,
248        Finding::warning(
249            IN_PROGRESS_CODE,
250            format!(
251                "the file is still being written (modified {:.1} s ago; missing: {}): {}{} planes complete; `ok` covers the complete part only, run `check` again when the acquisition ends",
252                acq.modified_ago_s,
253                if acq.missing.is_empty() {
254                    "nothing yet".into()
255                } else {
256                    acq.missing.join(", ")
257                },
258                acq.complete_planes,
259                expected
260            ),
261        ),
262    );
263}
264
265/// For formats that cannot be read while growing: when opening failed as corrupt and the file
266/// changed within the live window, say so in the error (the exit code stays 4).
267pub fn annotate_error(path: &Path, err: crate::Error) -> crate::Error {
268    let crate::Error::Corrupt {
269        format,
270        detail,
271        offset,
272    } = err
273    else {
274        return err;
275    };
276    let w = window().as_secs_f64();
277    let ago = std::fs::metadata(path)
278        .and_then(|m| m.modified())
279        .ok()
280        .map(seconds_since);
281    match ago {
282        Some(a) if w > 0.0 && a <= w && a >= -60.0 => crate::Error::Corrupt {
283            format,
284            detail: format!(
285                "{detail} [modified {a:.1} s ago: the file may still be being written; this reader cannot read it until the acquisition ends (`openreadout watch` reports dataset_complete)]"
286            ),
287            offset,
288        },
289        _ => crate::Error::Corrupt {
290            format,
291            detail,
292            offset,
293        },
294    }
295}
296
297/// The complete planes of `ws` as a set (`planes` on a file still being written reads only
298/// these).
299pub fn complete_set(ws: &WriteState) -> std::collections::HashSet<(u32, PlaneIndex)> {
300    ws.complete.iter().copied().collect()
301}
302
303#[cfg(test)]
304mod tests {
305    use super::*;
306
307    fn ws(tail: u64) -> WriteState {
308        let mut w = WriteState::new();
309        w.missing.push("chunk map".into());
310        w.complete.push((0, PlaneIndex::default()));
311        w.tail_bytes = tail;
312        w.unit_bytes = Some(4096);
313        w.modified = Some(SystemTime::now());
314        w
315    }
316
317    #[test]
318    fn recent_and_consistent_is_in_progress() {
319        let a = assess_with(Path::new("x"), &ws(100), Duration::from_secs(60)).unwrap();
320        assert_eq!(a.state, AcquisitionState::InProgress);
321        assert_eq!(a.complete_planes, 1);
322    }
323
324    #[test]
325    fn old_is_interrupted() {
326        let mut w = ws(100);
327        w.modified = Some(SystemTime::now() - Duration::from_secs(3600));
328        let a = assess_with(Path::new("x"), &w, Duration::from_secs(60)).unwrap();
329        assert_eq!(a.state, AcquisitionState::Interrupted);
330    }
331
332    #[test]
333    fn damage_or_huge_tail_is_not_an_acquisition() {
334        let mut w = ws(100);
335        w.append_consistent = false;
336        assert!(assess_with(Path::new("x"), &w, Duration::from_secs(60)).is_none());
337        let w = ws(1 << 30);
338        assert!(assess_with(Path::new("x"), &w, Duration::from_secs(60)).is_none());
339    }
340
341    #[test]
342    fn zero_window_disables() {
343        let a = assess_with(Path::new("x"), &ws(0), Duration::ZERO).unwrap();
344        assert_eq!(a.state, AcquisitionState::Interrupted);
345    }
346
347    #[test]
348    fn check_findings_downgraded_only_in_progress() {
349        let mut r = CheckReport::new("x", "nd2");
350        r.push(Finding::error("truncated", "chunk map missing"));
351        let a = assess_with(Path::new("x"), &ws(0), Duration::from_secs(60)).unwrap();
352        apply_to_check(&mut r, &a);
353        assert!(r.ok);
354        assert_eq!(r.findings[0].code, IN_PROGRESS_CODE);
355        assert!(r.findings[1].message.starts_with("in progress:"));
356    }
357}