flashraw_core/
integrity.rs

1//! Integrity of the sensor payload: structured truncation reports, decode warnings,
2//! and the range checks every format runs before it reads sensor bytes.
3//!
4//! A strip, tile or other sensor-data range that extends past the end of the file is
5//! corruption, and never silent. Under the recover policy (the default,
6//! [`DecodeContext::recover`]) the decoder uses every complete sample it has, fills the
7//! rest with zeros and records a [`DecodeWarning`]. Under the strict policy it fails the
8//! decode with an [`Error::CorruptFile`] whose [`CorruptReason::Truncated`] carries the
9//! expected range, the bytes actually present and the file length.
10//!
11//! Tolerances (data that is short but still complete):
12//! - Only the bytes the geometry needs are required: the last row needs its samples,
13//!   not its padding to the full stride, and a declared length beyond that need (end
14//!   padding) may run past the end of the file.
15//! - Formats addressed purely by geometry may read past a declared byte count that is
16//!   too small when the file holds the bytes (some writers understate it).
17
18use crate::{DecodeContext, Error, Result};
19use std::fmt;
20
21/// Which part of the sensor payload a report refers to.
22#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
23#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
24#[non_exhaustive]
25pub enum DataUnit {
26    /// The payload as a whole (formats that store it as one range).
27    Payload,
28    /// A TIFF strip (or an equivalent vendor strip), by index.
29    Strip(u32),
30    /// A tile, by index (row-major).
31    Tile(u32),
32    /// A separately stored plane, field or frame, by index.
33    Plane(u32),
34}
35
36impl DataUnit {
37    /// Index of the strip, tile or plane (`None` for [`DataUnit::Payload`]).
38    pub fn index(&self) -> Option<u32> {
39        match *self {
40            DataUnit::Payload => None,
41            DataUnit::Strip(i) | DataUnit::Tile(i) | DataUnit::Plane(i) => Some(i),
42        }
43    }
44
45    /// `"payload"`, `"strip"`, `"tile"` or `"plane"`.
46    pub fn kind(&self) -> &'static str {
47        match self {
48            DataUnit::Payload => "payload",
49            DataUnit::Strip(_) => "strip",
50            DataUnit::Tile(_) => "tile",
51            DataUnit::Plane(_) => "plane",
52        }
53    }
54}
55
56impl fmt::Display for DataUnit {
57    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
58        match self.index() {
59            Some(i) => write!(f, "{} {i}", self.kind()),
60            None => f.write_str("image data"),
61        }
62    }
63}
64
65/// A sensor-data range that the file does not fully contain.
66#[derive(Debug, Clone, PartialEq, Eq)]
67#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
68#[non_exhaustive]
69pub struct Truncation {
70    /// The strip, tile or plane affected.
71    pub unit: DataUnit,
72    /// File offset where the unit starts.
73    pub offset: u64,
74    /// Bytes the unit needs from `offset` (its geometry, or its declared length).
75    pub expected: u64,
76    /// Bytes present from `offset` to the end of the file (0 when `offset` is past it).
77    pub available: u64,
78    /// Length of the input.
79    pub file_len: u64,
80}
81
82impl Truncation {
83    /// A report for `expected` bytes at `offset` of a `file_len`-byte input.
84    pub fn new(unit: DataUnit, offset: u64, expected: u64, file_len: u64) -> Self {
85        let available = file_len.saturating_sub(offset).min(expected);
86        Truncation { unit, offset, expected, available, file_len }
87    }
88
89    /// Bytes missing from the unit.
90    pub fn missing(&self) -> u64 {
91        self.expected.saturating_sub(self.available)
92    }
93
94    /// True when the unit starts at or past the end of the file.
95    pub fn starts_beyond_eof(&self) -> bool {
96        self.offset >= self.file_len
97    }
98
99    /// True when `offset + expected` does not fit in the address space.
100    pub fn range_overflows(&self) -> bool {
101        self.offset.checked_add(self.expected).is_none_or(|e| usize::try_from(e).is_err())
102    }
103}
104
105impl fmt::Display for Truncation {
106    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
107        let unit = match self.unit {
108            DataUnit::Payload => String::new(),
109            u => format!("{u} "),
110        };
111        if self.range_overflows() {
112            write!(
113                f,
114                "truncated image data: {unit}range of {} bytes at offset {} overflows (file is {} bytes)",
115                self.expected, self.offset, self.file_len
116            )
117        } else if self.starts_beyond_eof() {
118            write!(
119                f,
120                "truncated image data: {unit}expected {} bytes at offset {}, which is beyond the end of the file ({} bytes)",
121                self.expected, self.offset, self.file_len
122            )
123        } else {
124            write!(
125                f,
126                "truncated image data: {unit}expected {} bytes at offset {}, but only {} bytes remain before EOF",
127                self.expected, self.offset, self.available
128            )
129        }
130    }
131}
132
133/// Structured cause of an [`Error::CorruptFile`] (or of a recovered [`DecodeWarning`]).
134#[derive(Debug, Clone, PartialEq, Eq)]
135#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
136#[non_exhaustive]
137pub enum CorruptReason {
138    /// Sensor data extends past the end of the file.
139    Truncated(Truncation),
140    /// A compressed unit could not be decoded (reported as a warning when recovered).
141    InvalidUnit {
142        /// The strip or tile affected.
143        unit: DataUnit,
144        /// What the decoder reported.
145        message: String,
146    },
147    /// The sensor data could not be decoded (e.g. an unsupported compression) and the
148    /// frame holds the embedded full-size preview instead. Only reported as a warning, when
149    /// the caller opted into the preview fallback.
150    PreviewFallback {
151        /// Why the sensor data was not decoded (the decoder's error).
152        message: String,
153    },
154}
155
156impl CorruptReason {
157    /// The strip, tile or plane affected.
158    pub fn unit(&self) -> DataUnit {
159        match self {
160            CorruptReason::Truncated(t) => t.unit,
161            CorruptReason::InvalidUnit { unit, .. } => *unit,
162            CorruptReason::PreviewFallback { .. } => DataUnit::Payload,
163        }
164    }
165
166    /// Machine-friendly kind: `"truncated"`, `"invalid-unit"` or `"preview-fallback"`.
167    pub fn kind(&self) -> &'static str {
168        match self {
169            CorruptReason::Truncated(_) => "truncated",
170            CorruptReason::InvalidUnit { .. } => "invalid-unit",
171            CorruptReason::PreviewFallback { .. } => "preview-fallback",
172        }
173    }
174
175    /// The truncation details, for [`CorruptReason::Truncated`].
176    pub fn truncation(&self) -> Option<&Truncation> {
177        match self {
178            CorruptReason::Truncated(t) => Some(t),
179            _ => None,
180        }
181    }
182}
183
184impl fmt::Display for CorruptReason {
185    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
186        match self {
187            CorruptReason::Truncated(t) => t.fmt(f),
188            CorruptReason::InvalidUnit { unit, message } => write!(f, "corrupt {unit}: {message}"),
189            CorruptReason::PreviewFallback { message } => write!(f, "sensor data not decoded: {message}"),
190        }
191    }
192}
193
194/// Something a decode repaired or substituted instead of failing: under the recover policy
195/// (truncated or corrupt units), or with the preview fallback (see
196/// [`CorruptReason::PreviewFallback`]).
197#[derive(Debug, Clone, PartialEq, Eq)]
198#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
199#[non_exhaustive]
200pub struct DecodeWarning {
201    /// What was wrong.
202    pub reason: CorruptReason,
203    /// Output samples that hold no decoded data (set to zero).
204    pub samples_filled: u64,
205}
206
207impl DecodeWarning {
208    /// A warning for `reason` with `samples_filled` zero-filled samples.
209    pub fn new(reason: CorruptReason, samples_filled: u64) -> Self {
210        DecodeWarning { reason, samples_filled }
211    }
212
213    /// Machine-friendly kind: `"truncated"`, `"invalid-unit"` or `"preview-fallback"`.
214    pub fn kind(&self) -> &'static str {
215        self.reason.kind()
216    }
217}
218
219impl fmt::Display for DecodeWarning {
220    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
221        match &self.reason {
222            CorruptReason::Truncated(t) => write!(
223                f,
224                "recovered truncated {}: {} / {} bytes available; {} missing samples were zero-filled",
225                t.unit, t.available, t.expected, self.samples_filled
226            ),
227            CorruptReason::InvalidUnit { unit, message } => {
228                write!(f, "recovered corrupt {unit} ({message}); its {} samples were zero-filled", self.samples_filled)
229            }
230            CorruptReason::PreviewFallback { message } => {
231                write!(f, "sensor data not decoded ({message}); the frame holds the embedded full-size preview")
232            }
233        }
234    }
235}
236
237/// Geometry of a row-structured payload, used to work out how many bytes are needed
238/// and how many samples a truncated range still holds.
239#[derive(Debug, Clone, Copy, PartialEq, Eq)]
240pub struct RowGeometry {
241    /// Rows in the unit.
242    pub rows: usize,
243    /// Bytes from one row start to the next.
244    pub stride: usize,
245    /// Bytes holding a row's samples (`<= stride`; the rest of the stride is padding).
246    pub row_bytes: usize,
247    /// Samples per row.
248    pub samples: usize,
249}
250
251impl RowGeometry {
252    /// Rows of `samples` values packed at `bits` per sample, `stride` bytes apart.
253    pub fn packed(rows: usize, samples: usize, bits: u32, stride: usize) -> Self {
254        let row_bytes = (samples.saturating_mul(bits as usize)).div_ceil(8);
255        RowGeometry { rows, stride: stride.max(row_bytes), row_bytes, samples }
256    }
257
258    /// Tightly packed rows.
259    pub fn tight(rows: usize, samples: usize, bits: u32) -> Self {
260        let row_bytes = (samples.saturating_mul(bits as usize)).div_ceil(8);
261        RowGeometry { rows, stride: row_bytes, row_bytes, samples }
262    }
263
264    /// Bytes needed for every sample: the last row needs no padding. Saturates.
265    pub fn needed(&self) -> usize {
266        if self.rows == 0 {
267            return 0;
268        }
269        (self.rows - 1).saturating_mul(self.stride).saturating_add(self.row_bytes)
270    }
271
272    /// Total samples.
273    pub fn total(&self) -> u64 {
274        self.rows as u64 * self.samples as u64
275    }
276
277    /// Rows whose samples lie entirely within the first `avail` bytes.
278    pub fn complete_rows(&self, avail: usize) -> usize {
279        if avail < self.row_bytes || self.stride == 0 {
280            return 0;
281        }
282        ((avail - self.row_bytes) / self.stride + 1).min(self.rows)
283    }
284
285    /// Samples entirely within the first `avail` bytes (complete rows plus the
286    /// complete samples of a partial row).
287    pub fn complete_samples(&self, avail: usize) -> u64 {
288        let rows = self.complete_rows(avail);
289        if rows >= self.rows {
290            return self.total();
291        }
292        let rest = avail.saturating_sub(rows * self.stride).min(self.row_bytes);
293        let partial = if self.row_bytes == 0 { 0 } else { rest as u64 * self.samples as u64 / self.row_bytes as u64 };
294        rows as u64 * self.samples as u64 + partial
295    }
296
297    /// Samples not covered by the first `avail` bytes.
298    pub fn missing_samples(&self, avail: usize) -> u64 {
299        self.total() - self.complete_samples(avail)
300    }
301}
302
303/// Check that `[offset, offset + needed)` lies within an input of `file_len` bytes.
304/// Nothing is missing from an empty range, wherever it starts.
305pub fn check_range(file_len: usize, unit: DataUnit, offset: usize, needed: usize) -> Result<(), Truncation> {
306    if needed == 0 {
307        return Ok(());
308    }
309    match offset.checked_add(needed) {
310        Some(end) if end <= file_len => Ok(()),
311        _ => Err(Truncation::new(unit, offset as u64, needed as u64, file_len as u64)),
312    }
313}
314
315/// Sensor bytes of one unit, as resolved by [`DecodeContext::sensor_bytes`].
316#[derive(Debug, Clone)]
317pub struct SensorBytes<'a> {
318    /// The unit's bytes: all `needed` bytes, or (recover policy only) what the file has.
319    pub bytes: &'a [u8],
320    /// Set when the unit is truncated (recover policy only): report it with
321    /// [`DecodeContext::recovered`] once the samples it lacks are known.
322    pub truncation: Option<Truncation>,
323}
324
325impl<'a> SensorBytes<'a> {
326    /// True when the unit is complete.
327    pub fn is_complete(&self) -> bool {
328        self.truncation.is_none()
329    }
330}
331
332impl DecodeContext<'_> {
333    /// The bytes `[offset, offset + needed)` of sensor-data `unit` in `data` (the whole
334    /// input). Under the strict policy a range not entirely in the file is an
335    /// [`Error::CorruptFile`] carrying a [`Truncation`]; under the recover policy the
336    /// available prefix (possibly empty) is returned with the truncation, which the caller
337    /// reports through [`DecodeContext::recovered`] after zero-filling what is missing.
338    pub fn sensor_bytes<'a>(
339        &self,
340        data: &'a [u8],
341        unit: DataUnit,
342        offset: usize,
343        needed: usize,
344    ) -> Result<SensorBytes<'a>> {
345        match check_range(data.len(), unit, offset, needed) {
346            Ok(()) => Ok(SensorBytes { bytes: data.get(offset..offset + needed).unwrap_or(&[]), truncation: None }),
347            Err(t) if self.recover => {
348                let start = offset.min(data.len());
349                let end = start.saturating_add(needed).min(data.len());
350                Ok(SensorBytes { bytes: &data[start..end], truncation: Some(t) })
351            }
352            Err(t) => Err(Error::truncated(t)),
353        }
354    }
355
356    /// Like [`DecodeContext::sensor_bytes`] for a row-structured unit: `needed` comes
357    /// from `geometry`, and a recovered truncation is reported immediately with the
358    /// samples the available bytes do not cover (callers must leave those zero).
359    pub fn sensor_rows<'a>(
360        &self,
361        data: &'a [u8],
362        unit: DataUnit,
363        offset: usize,
364        geometry: RowGeometry,
365    ) -> Result<&'a [u8]> {
366        let s = self.sensor_bytes(data, unit, offset, geometry.needed())?;
367        if let Some(t) = s.truncation {
368            let filled = geometry.missing_samples(s.bytes.len());
369            self.recovered(CorruptReason::Truncated(t), filled);
370        }
371        Ok(s.bytes)
372    }
373
374    /// Fail with `t` (strict), or record it as recovered with `samples_filled` zero
375    /// samples (recover). For checks made outside [`DecodeContext::sensor_bytes`].
376    pub fn truncated(&self, t: Truncation, samples_filled: u64) -> Result<()> {
377        if self.recover {
378            self.recovered(CorruptReason::Truncated(t), samples_filled);
379            Ok(())
380        } else {
381            Err(Error::truncated(t))
382        }
383    }
384
385    /// Record a problem the decode repaired (recover policy).
386    pub fn recovered(&self, reason: CorruptReason, samples_filled: u64) {
387        self.warnings.lock().unwrap_or_else(|p| p.into_inner()).push(DecodeWarning::new(reason, samples_filled));
388    }
389
390    /// Warnings recorded so far, sorted by unit (decoding may run units in parallel).
391    pub fn take_warnings(&self) -> Vec<DecodeWarning> {
392        let mut w = std::mem::take(&mut *self.warnings.lock().unwrap_or_else(|p| p.into_inner()));
393        w.sort_by_key(|w| {
394            let u = w.reason.unit();
395            (u.kind(), u.index())
396        });
397        w
398    }
399}
400
401#[cfg(test)]
402mod tests {
403    use super::*;
404    use crate::Profiler;
405
406    #[test]
407    fn messages() {
408        let t = Truncation::new(DataUnit::Strip(17), 24_576_000, 131_072, 24_576_000 + 48_291);
409        assert_eq!(t.available, 48_291);
410        assert_eq!(t.missing(), 131_072 - 48_291);
411        assert_eq!(
412            t.to_string(),
413            "truncated image data: strip 17 expected 131072 bytes at offset 24576000, but only 48291 bytes remain before EOF"
414        );
415        let e = Error::truncated(t.clone());
416        assert_eq!(e.kind(), "CorruptFile");
417        assert_eq!(e.truncation(), Some(&t));
418        assert!(e.to_string().starts_with("corrupt file: truncated image data: strip 17"));
419        let w = DecodeWarning::new(CorruptReason::Truncated(t), 10);
420        assert_eq!(
421            w.to_string(),
422            "recovered truncated strip 17: 48291 / 131072 bytes available; 10 missing samples were zero-filled"
423        );
424        let beyond = Truncation::new(DataUnit::Tile(2), 1000, 10, 500);
425        assert!(beyond.starts_beyond_eof());
426        assert_eq!(beyond.available, 0);
427        assert!(beyond.to_string().contains("beyond the end of the file"));
428        assert!(Truncation::new(DataUnit::Payload, u64::MAX - 1, 10, 5).range_overflows());
429    }
430
431    #[test]
432    fn geometry() {
433        // 3 rows of 4 12-bit samples (6 bytes) at a stride of 8.
434        let g = RowGeometry::packed(3, 4, 12, 8);
435        assert_eq!(g.needed(), 22);
436        assert_eq!(g.complete_rows(22), 3);
437        assert_eq!(g.complete_rows(21), 2);
438        assert_eq!(g.complete_samples(21), 8 + 3);
439        assert_eq!(g.missing_samples(0), 12);
440        assert_eq!(g.missing_samples(100), 0);
441    }
442
443    #[test]
444    fn policies() {
445        let p = Profiler::new();
446        let data = [0u8; 10];
447        let mut ctx = DecodeContext::new(&p);
448        assert!(ctx.recover, "recover is the default");
449        ctx.recover = false;
450        assert_eq!(ctx.sensor_bytes(&data, DataUnit::Payload, 2, 8).unwrap().bytes.len(), 8);
451        let e = ctx.sensor_bytes(&data, DataUnit::Strip(1), 4, 8).unwrap_err();
452        assert_eq!(e.truncation().unwrap().available, 6);
453        assert!(ctx.sensor_bytes(&data, DataUnit::Strip(1), usize::MAX, 8).is_err());
454        assert!(ctx.sensor_bytes(&data, DataUnit::Strip(1), 100, 0).unwrap().bytes.is_empty());
455        ctx.recover = true;
456        let s = ctx.sensor_bytes(&data, DataUnit::Strip(1), 4, 8).unwrap();
457        assert_eq!(s.bytes.len(), 6);
458        assert!(!s.is_complete());
459        let s = ctx.sensor_bytes(&data, DataUnit::Strip(2), usize::MAX, 8).unwrap();
460        assert!(s.bytes.is_empty());
461        assert_eq!(ctx.sensor_rows(&data, DataUnit::Tile(0), 4, RowGeometry::tight(2, 4, 8)).unwrap().len(), 6);
462        let w = ctx.take_warnings();
463        assert_eq!(w.len(), 1);
464        assert_eq!(w[0].samples_filled, 2);
465    }
466}
flashraw_core/integrity.rs

flashraw_core/
integrity.rs
