medh5 2.0.0

The MEDH5 format engine: a self-describing HDF5 container for one medical imaging subject and all of its ground truth
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215

//! Source references: a locator, an optional member key, and a pin.
//!
//! A URI says where bytes are; it is not what they are.  The pinned
//! `content_id` is: a reference resolves to a sample, and the sample counts
//! only while its address is the pinned one.  Since `content_id` is a Merkle
//! root over *stored* digests, the check also recomputes the root and
//! re-verifies the clinical datasets' actual bytes --- a stored root alone
//! would miss an edit under unchanged digests (1.1 §8).

use std::path::{Path, PathBuf};

use serde_json::{json, Value};

use super::Finding;
use crate::collection::{open_any, AnyFile};
use crate::json::repr_str;
use crate::sample::Sample;
use crate::{Error, Result};

/// One sample, standalone or a collection member, at a pinned version.
#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub struct SourceRef {
    /// The id rows and reconciliation records use for this source.
    pub source_id: String,
    /// A path, relative to the manifest's directory, or absolute; `file://`
    /// URIs are accepted.
    pub uri: String,
    /// The member of a `.medh5c` collection, or `None`.
    pub sample_key: Option<String>,
    /// The sample version the reference pins.
    pub content_id: String,
    /// The sample's own `identity.subject_id`, as the manifest maps it.
    pub local_subject_id: Option<String>,
}

impl SourceRef {
    pub fn new(uri: impl Into<String>, sample_key: Option<String>, content_id: impl Into<String>) -> SourceRef {
        let uri = uri.into();
        SourceRef { source_id: String::new(), uri, sample_key, content_id: content_id.into(), local_subject_id: None }
    }

    /// A reference pinned to what `sample` is now.
    ///
    /// Refused when the root does not bind every attested path to its bytes
    /// (1.0 §13.2): such a pin could never check clean, and a file crafted
    /// with an alias holding an attested name --- relinkable afterwards to
    /// other covered bytes under the same root --- would pin without
    /// complaint.
    pub fn pin(uri: impl Into<String>, sample_key: Option<String>, sample: &Sample) -> Result<SourceRef> {
        let content_id = sample
            .content_id()?
            .ok_or_else(|| Error::invalid("a sample without a content_id cannot be pinned; commit it first"))?;
        let mut out = SourceRef::new(uri, sample_key, content_id);
        let uncovered = crate::integrity::unattested(&sample.root)?;
        if !uncovered.is_empty() {
            return Err(Error::Integrity(format!(
                "{} cannot be pinned: no line of its content_id binds {} to {} bytes (1.0 §13.2); `medh5 validate \
                 --level integrity` says why",
                out.locator(),
                uncovered.join(", "),
                if uncovered.len() == 1 { "its" } else { "their" },
            )));
        }
        out.local_subject_id = Some(sample.identity()?.subject_id.clone());
        Ok(out)
    }

    /// The file the locator names, relative to `base` when it is relative.
    pub fn resolve(&self, base: Option<&Path>) -> PathBuf {
        let text = self.uri.strip_prefix("file://").unwrap_or(&self.uri);
        let path = PathBuf::from(text);
        match base {
            Some(b) if path.is_relative() => b.join(path),
            _ => path,
        }
    }

    /// `uri` or `uri::key`, for messages.
    pub fn locator(&self) -> String {
        match &self.sample_key {
            Some(k) => format!("{}::{k}", self.uri),
            None => self.uri.clone(),
        }
    }

    /// Open the sample the reference names (not yet checking the pin).
    pub fn open(&self, base: Option<&Path>) -> Result<Sample> {
        let path = self.resolve(base);
        match open_any(&path, self.sample_key.as_deref())? {
            AnyFile::Sample(s) => Ok(s),
            AnyFile::Collection(_) => Err(Error::invalid(format!(
                "{} is a collection; a source names one of its members with `sample_key`",
                repr_str(&self.uri)
            ))),
        }
    }

    /// Whether `sample` is the pinned version: its stored `content_id` is the
    /// pin, every dataset the root speaks for carries a digest, the root
    /// recomputes to it, and every clinical dataset's bytes match their
    /// digests.  `deep` verifies every dataset.  Empty when it is.
    ///
    /// The sample is walked once: the stored digests the root is recomputed
    /// from are the ones the bytes are checked against.  A dataset whose
    /// bytes cannot be read --- a damaged chunk --- does not match.
    pub fn check(&self, sample: &Sample, deep: bool) -> Result<Vec<Finding>> {
        let mut out = Vec::new();
        let at = if self.source_id.is_empty() { self.locator() } else { self.source_id.clone() };
        let stored = sample.content_id()?;
        if stored.as_deref() != Some(self.content_id.as_str()) {
            out.push(Finding::new(
                "T302",
                &at,
                format!(
                    "{} is now {}; the reference pins {} --- repin it explicitly if the change is meant",
                    self.locator(),
                    stored.as_deref().unwrap_or("unaddressed"),
                    self.content_id
                ),
            ));
            return Ok(out);
        }
        let digests = crate::integrity::collect_digests(&sample.root, &["index"])?;
        // The root covers stored digests at the paths their lines name, so a
        // dataset that carries none --- a column added later, say --- is bytes
        // no pin speaks for, and so is one reached at a path that is not its
        // own: refused before anything is recomputed (1.0 §13.2, 1.1 §8,
        // E818).  Every path through the attested groups is judged, soft links
        // followed as readers follow them: an object listed under no line ---
        // reached first through `index/`, or only through a soft link --- is
        // no more covered than one without a digest, and a name held by
        // an alias could be relinked to other covered bytes under the same
        // root.
        let uncovered = crate::integrity::unattested(&sample.root)?;
        if !uncovered.is_empty() {
            out.push(Finding::new(
                "T302",
                &at,
                format!(
                    "{}: no line of the root binds {} to {} bytes, so the pinned content_id does not cover {}",
                    self.locator(),
                    uncovered.join(", "),
                    if uncovered.len() == 1 { "its" } else { "their" },
                    if uncovered.len() == 1 { "it" } else { "them" }
                ),
            ));
            return Ok(out);
        }
        let algo = crate::integrity::digest::root_algo(&sample.root)?;
        let recomputed =
            crate::integrity::compute_content_id(&sample.root, &sample.attr_name_map()?, &algo, Some(&digests))?;
        if recomputed != self.content_id {
            out.push(Finding::new(
                "T302",
                &at,
                format!("{} declares the pinned content_id but its digests recompute to {recomputed}", self.locator()),
            ));
            return Ok(out);
        }
        // Every path through `clinical/` is its object's own, checked above,
        // so the clinical datasets are the lines under it.
        let mut failed = Vec::new();
        for (path, digest) in &digests {
            if !deep && !path.starts_with("clinical/") {
                continue;
            }
            let matches = crate::digest::parse_digest(digest).and_then(|(algo, _)| {
                let ds = sample.root.dataset(path)?;
                Ok(&crate::integrity::dataset_digest(&ds, path, &algo)? == digest)
            });
            if !matches.unwrap_or(false) {
                failed.push(path.clone());
            }
        }
        if !failed.is_empty() {
            failed.sort();
            out.push(Finding::new(
                "T302",
                &at,
                format!(
                    "{}: dataset bytes no longer match their digests ({}), whatever the stored root says",
                    self.locator(),
                    failed.join(", ")
                ),
            ));
        }
        Ok(out)
    }

    pub fn to_json(&self) -> Value {
        json!({
            "source_id": self.source_id,
            "uri": self.uri,
            "sample_key": self.sample_key,
            "content_id": self.content_id,
            "local_subject_id": self.local_subject_id,
        })
    }

    /// The `{uri, sample_key, content_id}` a cache entry records.
    pub fn to_pin_json(&self) -> Value {
        json!({"uri": self.uri, "sample_key": self.sample_key, "content_id": self.content_id})
    }

    pub fn from_json(value: &Value) -> Result<SourceRef> {
        let text = |k: &str| value.get(k).and_then(Value::as_str).map(str::to_string);
        Ok(SourceRef {
            source_id: text("source_id").unwrap_or_default(),
            uri: text("uri").ok_or_else(|| Error::invalid("a source names its `uri`"))?,
            sample_key: text("sample_key"),
            content_id: text("content_id").ok_or_else(|| Error::invalid("a source pins its `content_id`"))?,
            local_subject_id: text("local_subject_id"),
        })
    }
}