acorn/io/
bagit.rs

1//! Support for working with [BagIt](https://tools.ietf.org/html/rfc8493) format data.
2//!
3//! BagIt is a standardized file packaging format designed for digital preservation and data transfer.
4//! This module provides utilities to create, verify, and work with BagIt packages containing research
5//! activity data, metadata, and supporting documentation.
6//!
7use crate::io::{archive, file_checksum, files_all, read_file, write_file, ApiResult, StringConversion};
8use crate::prelude::{copy, create_dir_all, PathBuf};
9use crate::util::{ChecksumAlgorithm, Label};
10use bon::Builder;
11use color_eyre::eyre::eyre;
12use core::fmt;
13use derive_more::Display;
14use schemars::JsonSchema;
15use serde::{Deserialize, Serialize};
16use tracing::{error, warn};
17
18const LINE_ENDING: &str = if cfg!(windows) { "\r\n" } else { "\n" };
19/// Trait for persisting BagIt data
20pub trait Save {
21    /// Persist BagIt data to a given path
22    fn save<P>(&self, destination: P) -> ApiResult<PathBuf>
23    where
24        P: Into<PathBuf> + Clone;
25}
26/// Container structure for details of BagIt formatted data.
27///
28/// A bag consists of a base directory containing payload files in a `data/` subdirectory,
29/// along with manifest files and other tag files that provide metadata and checksums for
30/// validation. This struct represents the core bag configuration defined in `bagit.txt`.
31///
32/// See <https://datatracker.ietf.org/doc/html/draft-kunze-bagit> for details
33#[derive(Builder, Clone, Debug, Serialize, Deserialize, JsonSchema)]
34#[builder(start_fn = init)]
35pub struct Bag {
36    /// Root directory path of the bag containing all bag files and directories
37    pub base_directory: String,
38    /// List of payload file paths relative to the bag's data/ directory
39    #[builder(default = Vec::new())]
40    pub payload: Vec<String>,
41    /// Checksum algorithm used in payload and tag manifest files
42    #[builder(default)]
43    pub checksum_algorithm: ChecksumAlgorithm,
44    /// BagIt specification version (e.g., "1.0")
45    #[builder(default = String::from("1.0"))]
46    pub version: String,
47    /// Character encoding for tag files (recommended: "UTF-8")
48    #[builder(default = String::from("UTF-8"))]
49    pub encoding: String,
50    /// Bag-Info metadata from `bag-info.txt`
51    pub info: Option<BagInfo>,
52}
53/// Metadata describing the bag and its payload from `bag-info.txt`.
54///
55/// Bag-Info metadata elements are intended primarily for human use and are optional per the BagIt
56/// specification. These fields correspond to the reserved metadata element names from RFC 8493.
57///
58/// See <https://datatracker.ietf.org/doc/html/draft-kunze-bagit> for details
59#[derive(Builder, Clone, Debug, Display, Serialize, Deserialize, JsonSchema)]
60#[display("Bag-Info")]
61#[builder(start_fn = init)]
62pub struct BagInfo {
63    /// Organization transferring the content (Source-Organization)
64    pub organization: Option<Vec<String>>,
65    /// Mailing address of the source organization (Organization-Address)
66    pub organization_address: Option<Vec<String>>,
67    /// Person responsible for the content transfer (Contact-Name)
68    pub contact_name: Option<Vec<String>>,
69    /// International format telephone number of responsible person (Contact-Phone)
70    pub contact_phone: Option<Vec<String>>,
71    /// Fully qualified email address of responsible person (Contact-Email)
72    pub contact_email: Option<Vec<String>>,
73    /// Brief explanation of contents and provenance (External-Description)
74    pub description: Option<Vec<String>>,
75    /// Date (YYYY-MM-DD) that content was prepared for transfer (Bagging-Date)
76    pub date: Option<String>,
77    /// Sender-supplied identifier for the bag (External-Identifier)
78    pub identifier: Option<Vec<String>>,
79    /// Size or approximate size of the bag with abbreviation (Bag-Size)
80    pub size: Option<String>,
81    /// Bag count values as (N, T) tuples where N is ordinal position and T is total
82    /// T is None if unknown ("?")
83    pub count: Option<Vec<(u32, Option<u32>)>>,
84}
85impl Bag {
86    /// Verify the completeness and correctness of a bag
87    /// ### Checks
88    /// - `bagit.txt` exists
89    /// - `manifest-<algorithm>.txt` exists
90    /// - `data/` exists
91    /// - Checksum values in manifest file matches checksum values of files in `data/`
92    pub fn verify<P>(path: P) -> ApiResult<()>
93    where
94        P: Into<PathBuf>,
95    {
96        let base_directory = path.into();
97        let files = files_all(base_directory.clone(), None)
98            .into_iter()
99            .filter(|x| x.is_file())
100            .collect::<Vec<_>>();
101        match files
102            .iter()
103            .flat_map(|x| x.file_name())
104            .flat_map(|x| x.to_str())
105            .find(|x| x.starts_with("manifest"))
106        {
107            | Some(name) => {
108                let algorithm = name["manifest.".len()..]
109                    .split(".")
110                    .collect::<Vec<_>>()
111                    .first()
112                    .unwrap_or(&"")
113                    .to_lowercase();
114                let manifest = format!("manifest-{}.txt", algorithm.clone());
115                let checksum_algorithm: &'static ring::digest::Algorithm = ChecksumAlgorithm::from(algorithm).into();
116                let required = ["bagit.txt", &manifest, "data/"]
117                    .iter()
118                    .map(|x| base_directory.join(x))
119                    .collect::<Vec<_>>();
120                if required.iter().all(|x| x.exists()) {
121                    let manifest_path = base_directory.join(manifest);
122                    let content = read_file(manifest_path.clone()).unwrap_or_default();
123                    let pairs = content
124                        .lines()
125                        .map(|line| line.split_whitespace().collect::<Vec<_>>())
126                        .collect::<Vec<_>>();
127                    let compare_checksums = pairs
128                        .iter()
129                        .map(|x| {
130                            let checksum = x.first().copied().unwrap_or("");
131                            let payload_path = base_directory.join(x.get(1).copied().unwrap_or(""));
132                            let calculated = file_checksum(payload_path, Some(checksum_algorithm))
133                                .map(|checksum| checksum.checksum_value)
134                                .unwrap_or_default();
135                            calculated.eq(checksum)
136                        })
137                        .collect::<Vec<_>>();
138                    match compare_checksums.iter().enumerate().find(|(_, result)| !*result) {
139                        | Some((index, _)) => {
140                            let path = pairs.get(index).and_then(|p| p.get(1)).copied().unwrap_or("");
141                            Err(eyre!("Checksum mismatch in payload file = {}", path))
142                        }
143                        | None => Ok(()),
144                    }
145                } else {
146                    Err(eyre!("Missing required file(s) and/or folder(s)"))
147                }
148            }
149            | None => Err(eyre!("No manifest file found")),
150        }
151    }
152    /// Populate payload file listing from files present in self.base_directory "data" directory
153    pub fn with_payload(&self) -> Self {
154        let base_directory = PathBuf::from(self.base_directory.clone());
155        let payload = files_all(base_directory.clone(), None)
156            .into_iter()
157            .filter(|x| x.is_file())
158            .flat_map(|x| x.strip_prefix(base_directory.to_absolute_string()).ok().map(|p| p.to_path_buf()))
159            .map(|x| x.display().to_string())
160            .collect::<Vec<_>>();
161        let Bag {
162            checksum_algorithm,
163            version,
164            encoding,
165            info,
166            ..
167        } = self;
168        Bag::init()
169            .base_directory(self.base_directory.clone())
170            .checksum_algorithm(checksum_algorithm.clone())
171            .version(version.clone())
172            .encoding(encoding.clone())
173            .maybe_info(info.clone())
174            .payload(payload)
175            .build()
176    }
177}
178impl Save for Bag {
179    /// Persist files in a given directory as a archived ("zipped") BagIt format package
180    fn save<P>(&self, destination: P) -> ApiResult<PathBuf>
181    where
182        P: Into<PathBuf> + Clone,
183    {
184        let Bag { info, version, encoding, .. } = self;
185        let dest = destination.clone().into();
186        let payload_directory = dest.join("data");
187        match create_dir_all(payload_directory.clone()) {
188            | Ok(_) => {
189                let base = PathBuf::from(self.base_directory.clone());
190                let bag = self.with_payload();
191                let checksum_algorithm: &'static ring::digest::Algorithm = self.checksum_algorithm.clone().into();
192                bag.payload.clone().into_iter().for_each(|x| {
193                    let payload = PathBuf::from(x);
194                    let from = base.join(&payload);
195                    let to = dest.join("data").join(&payload);
196                    if let Some(parent) = to.parent() {
197                        if let Err(why) = create_dir_all(parent) {
198                            error!(
199                                directory = parent.to_path_buf().to_absolute_string(),
200                                "=> {} Create - {why}",
201                                Label::fail()
202                            );
203                            return;
204                        }
205                        if let Err(why) = copy(from.clone(), to.clone()) {
206                            error!(
207                                from = from.to_absolute_string(),
208                                to = to.to_absolute_string(),
209                                "=> {} Copy - {why}",
210                                Label::fail()
211                            );
212                        }
213                    }
214                });
215                let bag_declaration_content = format!("BagIt-Version: {version}\nTag-File-Character-Encoding: {encoding}\n");
216                let payload_manifest_content = bag
217                    .payload
218                    .into_iter()
219                    .fold(String::new(), |acc, x| {
220                        let payload = PathBuf::from(x.clone());
221                        let to = dest.join("data").join(&payload);
222                        match file_checksum(to.clone(), Some(checksum_algorithm)) {
223                            | Some(checksum) => {
224                                format!("{acc}{checksum} data/{x}{LINE_ENDING}")
225                            }
226                            | None => {
227                                warn!(payload = payload.to_absolute_string(), "=> {} Calculate checksum", Label::fail());
228                                acc
229                            }
230                        }
231                    })
232                    .replace("\\", "/");
233                write_file(dest.clone().join("bagit.txt"), bag_declaration_content)
234                    .map(|_| dest.clone())
235                    .and_then(|_| match info.clone() {
236                        | Some(bag_info) => bag_info.save(destination.clone()),
237                        | None => Ok(dest.clone()),
238                    })
239                    .and_then(|_| {
240                        let file_name = format!("manifest-{}.txt", self.checksum_algorithm);
241                        let file_path = dest.clone().join(&file_name);
242                        write_file(file_path, payload_manifest_content).map(|_| dest.clone())
243                    })
244                    .and_then(|_| archive(dest.clone(), None))
245            }
246            | Err(why) => Err(eyre!("Failed to create bag - {why}")),
247        }
248    }
249}
250impl fmt::Display for Bag {
251    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
252        write!(f, "Bag v{} ({}): {}", self.version, self.checksum_algorithm, self.base_directory)
253    }
254}
255impl BagInfo {
256    /// Enumerate BagIt reserved metadata elements as ordered key/value pairs
257    pub fn entries(&self) -> Vec<(String, String)> {
258        let BagInfo {
259            organization,
260            organization_address,
261            contact_name,
262            contact_phone,
263            contact_email,
264            description,
265            date,
266            identifier,
267            size,
268            count,
269        } = self;
270        let repeatable = [
271            ("Source-Organization", organization),
272            ("Organization-Address", organization_address),
273            ("Contact-Name", contact_name),
274            ("Contact-Phone", contact_phone),
275            ("Contact-Email", contact_email),
276            ("External-Description", description),
277            ("External-Identifier", identifier),
278        ]
279        .into_iter()
280        .flat_map(|(key, values)| {
281            values
282                .iter()
283                .flat_map(|items| items.iter())
284                .map(move |value| (key.to_string(), value.clone()))
285        });
286        let count = count.iter().flat_map(|items| items.iter()).map(|(index, total)| {
287            let total = total.map_or_else(|| "?".to_string(), |value| value.to_string());
288            ("Bag-Count".to_string(), format!("{index} of {total}"))
289        });
290        let single = [("Bagging-Date", date.clone()), ("Bag-Size", size.clone())]
291            .into_iter()
292            .filter_map(|(key, value)| value.map(|v| (key.to_string(), v)));
293        repeatable.chain(count).chain(single).collect()
294    }
295}
296impl Default for BagInfo {
297    fn default() -> Self {
298        BagInfo::init().build()
299    }
300}
301impl Save for BagInfo {
302    fn save<P>(&self, destination: P) -> ApiResult<PathBuf>
303    where
304        P: Into<PathBuf> + Clone,
305    {
306        let dest = destination.clone().into();
307        let content = self
308            .entries()
309            .into_iter()
310            .fold(String::new(), |acc, (key, value)| format!("{acc}{key}: {value}{LINE_ENDING}"));
311        write_file(dest.join("bag-info.txt"), content).map(|_| dest)
312    }
313}
314impl From<ChecksumAlgorithm> for &'static ring::digest::Algorithm {
315    fn from(value: ChecksumAlgorithm) -> Self {
316        match value {
317            | ChecksumAlgorithm::Sha256 => &ring::digest::SHA256,
318            | ChecksumAlgorithm::Sha512 => &ring::digest::SHA512,
319            | _ => &ring::digest::SHA256,
320        }
321    }
322}
acorn/io/bagit.rs

acorn/io/
bagit.rs
