file_with_meta/

lib.rs

#![warn(missing_docs)]
// SPDX-FileCopyrightText: Peter Pentchev <roam@ringlet.net>
// SPDX-License-Identifier: BSD-2-Clause
//! Store a file's metadata for caching purposes.
//!
//! The [`FileHttpMetadata`] structure may be serialized and
//! stored in a JSON file alongside the real file, e.g. one with
//! ".meta" appended to the file name. Then either the [`match_meta`]
//! function may be used directly, or the [`build_req`] one may be
//! used to modify an HTTP request, adding the necessary headers to
//! make sure that the file is not downloaded if there have been
//! no changes on the remote server.
//!
//! Example for checking whether a file needs to be downloaded:
//! ```rust
//! # use std::error::Error;
//! use std::fs::{self, File};
//! # use std::io::{self, BufWriter, Read, Write};
//! # use std::path::Path;
//!
//! # use anyhow::{Context as _, Result};
//! # use camino::Utf8Path;
//!
//! # #[cfg(feature = "ureq")]
//! # fn main() -> Result<()> {
//! # let agent = ureq::agent();
//! # let tempd_obj = tempfile::tempdir()?;
//! # let destdir: &Utf8Path = tempd_obj
//! #     .as_ref()
//! #     .try_into()
//! #     .with_context(|| format!("Could not represent {tempd_obj:?} as a UTF-8 path"))?;
//! let dst = destdir.join("data.json");
//! let dst_meta = destdir.join("data.json.meta");
//! let (req, stored_meta) = file_with_meta::build_req(
//!     agent.get("https://example.com/"),
//!     &dst,
//!     &dst_meta,
//! )?;
//! let resp = req.call()?;
//! match resp.status() {
//!     304 => println!("Nothing was fetched"),
//!     _ => {
//!         println!("Storing the content");
//!         /* ... */
//! #         let mut reader = resp.into_reader();
//! #         let mut outfile = File::create(&dst)?;
//! #         let mut writer = BufWriter::new(&outfile);
//! #         loop {
//! #             let mut buf = [0; 8192];
//! #             let n = reader.read(&mut buf[..])?;
//! #             if n == 0 {
//! #                 break;
//! #             }
//! #             writer.write_all(&buf[..n])?;
//! #         }
//! #         writer.flush()?;
//! #         outfile.sync_all()?;
//!
//!         println!("Updating the file's metadata");
//!         let meta = file_with_meta::FileHttpMetadata::from_file(&dst)?;
//!         fs::write(&dst_meta, serde_json::to_string(&meta).unwrap())?;
//!     }
//! };
//! # Ok(())
//! # }
//! # #[cfg(not(feature = "ureq"))]
//! # fn main() {
//! # }
//! ```
//!
//! Example for checking whether a file has changed since its metadata
//! was last updated:
//! ```rust
//! let dst = "/path/to/file.dat";
//! let dst_meta = "/path/to/file.dat.meta";
//!
//! match file_with_meta::match_meta(&dst, &dst_meta)?.is_some() {
//!     true => println!("No change"),
//!     false => println!("Somebody touched our file, recreate it?"),
//! };
//! # Ok::<_, file_with_meta::Error>(())
//! ```
//!
//! The [`match_meta_with_source`] function may be used to additionally
//! make sure that a "source" file has not been modified since this file
//! was last generated from its data.

#![doc(html_root_url = "https://docs.rs/file-with-meta/0.3.0")]
use std::fs::{self, Metadata};
use std::io::Error as IoError;
use std::time::SystemTime;

use camino::Utf8Path;
use serde_derive::{Deserialize, Serialize};
use serde_json::Error as SJError;
use thiserror::Error;

#[cfg(feature = "ureq")]
use ureq::Request;

#[cfg(test)]
mod tests;

/// An error that occurred during processing the metadata.
#[derive(Debug, Error)]
#[non_exhaustive]
#[allow(clippy::error_impl_error)]
pub enum Error {
    /// Could not examine a local file.
    #[error("Could not examine {0}")]
    Examine(String, #[source] IoError),

    /// Unexpected format major version in the metadata JSON structure.
    #[error("Unsupported format major version {0}")]
    FormatVersionMajor(u32),

    /// Something went really, really wrong...
    #[error("file-with-meta internal error: {0}")]
    Internal(String),

    /// Could not parse the metadata JSON structure.
    #[error("Could not parse the metadata")]
    Parse(SJError),
}

/// The version of the format of the serialized metadata.
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
pub struct MetadataFormatVersion {
    /// The major version number; bumped when a field is removed or
    /// its type is changed.
    major: u32,
    /// The minor version number; bumped when a new field is added.
    minor: u32,
}

impl Default for MetadataFormatVersion {
    /// The default format version is the most recent one.
    fn default() -> Self {
        Self { major: 0, minor: 1 }
    }
}

/// Information about the format of the JSON-serialized metadata.
#[derive(Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
pub struct MetadataFormat {
    /// The version of the metadata format, currently 0.x.
    version: MetadataFormatVersion,
}

#[derive(Debug, Serialize, Deserialize)]
struct MetadataTopLevelFormatOnly {
    format: MetadataFormat,
}

/// Information about a single file's last modification time and,
/// if specified, some relevant HTTP headers returned by the server
/// that the file was fetched from.
#[derive(Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
#[non_exhaustive]
pub struct FileHttpMetadata {
    /// The version of the metadata as stored in a JSON string.
    pub format: MetadataFormat,
    /// The size of the file.
    pub file_size: u64,
    /// The modification time of the file as a Unix timestamp.
    pub file_mtime: u64,
    /// The `Last-Modified` header as returned by an HTTP server.
    pub hdr_last_modified: Option<String>,
    /// The `ETag` header as returned by an HTTP server.
    pub hdr_etag: Option<String>,
    /// The size of the source file if applicable.
    pub source_file_size: Option<u64>,
    /// The modification time of the source file if applicable.
    pub source_file_mtime: Option<u64>,
    /// A hook for external users to store information about whether
    /// the file's contents has been validated.
    pub verified: bool,
}

impl FileHttpMetadata {
    /// Examine an existing file and return a metadata structure
    /// recording its size and last modification time.
    ///
    /// # Errors
    ///
    /// [`Error::Examine`] if the file cannot be examined.
    pub fn from_file<P>(path: P) -> Result<Self, Error>
    where
        P: AsRef<Utf8Path>,
    {
        let meta = fs::metadata(path.as_ref())
            .map_err(|err| Error::Examine(path.as_ref().to_string(), err))?;
        Ok(Self {
            file_size: meta.len(),
            file_mtime: mtime_to_unix(&meta)?,
            ..Self::default()
        })
    }

    /// Examine an existing file and return a metadata structure
    /// recording its size and last modification time, as well as
    /// that of the specified "source" file.
    ///
    /// # Errors
    ///
    /// Propagates errors from [`Self::from_file`] and [`mtime_to_unix`].
    /// [`Error::Examine`] if the source file cannot be examined.
    pub fn from_file_with_source<P1, P2>(path: P1, src: P2) -> Result<Self, Error>
    where
        P1: AsRef<Utf8Path>,
        P2: AsRef<Utf8Path>,
    {
        let meta = Self::from_file(path)?;
        let src_meta = fs::metadata(src.as_ref())
            .map_err(|err| Error::Examine(src.as_ref().to_string(), err))?;
        Ok(Self {
            source_file_size: Some(src_meta.len()),
            source_file_mtime: Some(mtime_to_unix(&src_meta)?),
            ..meta
        })
    }

    /// Examine an existing file and return a metadata structure
    /// recording its size and last modification time, as well as
    /// the previously-stored one for a "source" file.
    ///
    /// # Errors
    ///
    /// Propagates errors from [`Self::from_file`].
    pub fn from_file_with_source_meta<P>(path: P, src_meta: &Self) -> Result<Self, Error>
    where
        P: AsRef<Utf8Path>,
    {
        let meta = Self::from_file(path)?;
        Ok(Self {
            source_file_size: Some(src_meta.file_size),
            source_file_mtime: Some(src_meta.file_mtime),
            ..meta
        })
    }

    /// Parse a metadata structure from the supplied JSON string.
    /// Verify the version specified in the "format" element, do not
    /// even attempt to parse unknown versions.
    ///
    /// # Errors
    ///
    /// [`Error::Parse`] if the JSON data cannot be parsed.
    /// [`Error::FormatVersionMajor`] on unexpected format.version.major values.
    pub fn parse(contents: &str) -> Result<Self, Error> {
        let header =
            serde_json::from_str::<MetadataTopLevelFormatOnly>(contents).map_err(Error::Parse)?;
        match header.format.version.major {
            0 => serde_json::from_str::<Self>(contents).map_err(Error::Parse),
            _ => Err(Error::FormatVersionMajor(header.format.version.major)),
        }
    }
}

/// Unwrap a [`Metadata`] object's last modified timestamp,
/// assume it may be converted to a Unix timestamp, and return
/// the number of seconds since the Unix epoch.
///
/// # Errors
///
/// [`Error::Internal`] if the mtime cannot be fetche out of the metadata or
/// it cannot be converted to a duration since the Unix epoch.
pub fn mtime_to_unix(metadata: &Metadata) -> Result<u64, Error> {
    Ok(metadata
        .modified()
        .map_err(|err| {
            Error::Internal(format!("Could not get the mtime from {metadata:?}: {err}"))
        })?
        .duration_since(SystemTime::UNIX_EPOCH)
        .map_err(|err| {
            Error::Internal(format!(
                "Could not get a Unix timestamp from the 'modified' time in {metadata:?}: {err}"
            ))
        })?
        .as_secs())
}

/// Verify that a file has not been changed since the last time
/// the metadata was stored.
///
/// # Errors
///
/// Propagates errors from [`mtime_to_unix`].
// No need for MSRV 1.62
pub fn match_meta<P1, P2>(dst: P1, dst_meta: P2) -> Result<Option<FileHttpMetadata>, Error>
where
    P1: AsRef<Utf8Path>,
    P2: AsRef<Utf8Path>,
{
    if let Ok(file_meta) = fs::metadata(dst.as_ref()) {
        if let Ok(contents) = fs::read_to_string(dst_meta.as_ref()) {
            if let Ok(meta) = FileHttpMetadata::parse(&contents) {
                return Ok((file_meta.is_file()
                    && file_meta.len() == meta.file_size
                    && mtime_to_unix(&file_meta)? == meta.file_mtime)
                    .then_some(meta));
            }
        }
    }
    Ok(None)
}

/// Verify that a file has not been changed, and additionally verify
/// that its source file, specified by the `src` local path, has
/// also not been changed.
///
/// Useful when e.g. uncompressing or otherwise processing downloaded files.
///
/// # Errors
///
/// Propagates errors from [`match_meta`].
// No need for MSRV 1.62
pub fn match_meta_with_source<P1, P2, P3>(
    dst: P1,
    dst_meta: P2,
    src: P3,
) -> Result<Option<FileHttpMetadata>, Error>
where
    P1: AsRef<Utf8Path>,
    P2: AsRef<Utf8Path>,
    P3: AsRef<Utf8Path>,
{
    if let Some(meta) = match_meta(dst, dst_meta)? {
        Ok(match fs::metadata(src.as_ref()) {
            Ok(src_meta) => {
                let src_len = src_meta.len();
                if meta.source_file_size.unwrap_or(src_len) == src_len {
                    let src_mtime = mtime_to_unix(&src_meta)?;
                    (meta.source_file_mtime.unwrap_or(src_mtime) == src_mtime).then_some(meta)
                } else {
                    None
                }
            }
            Err(_) => (meta.source_file_size.is_none() && meta.source_file_mtime.is_none())
                .then_some(meta),
        })
    } else {
        Ok(None)
    }
}

#[cfg(feature = "ureq")]
#[allow(clippy::doc_markdown)]
/// Add the `If-Modified-Since` and/or `If-None-Match` headers to
/// an HTTP request if needed.
///
/// The headers will be added if the relevant fields (`Last-Modified` and `ETag`
/// respectively) have been returned in the last response from
/// the server when the file has been downloaded.
///
/// # Errors
///
/// Propagates errors from [`match_meta`].
pub fn build_req<P1, P2>(
    orig_req: Request,
    dst: P1,
    dst_meta: P2,
) -> Result<(Request, Option<FileHttpMetadata>), Error>
where
    P1: AsRef<Utf8Path>,
    P2: AsRef<Utf8Path>,
{
    let stored_meta = match_meta(dst, dst_meta)?;

    let req = match stored_meta {
        None => orig_req,
        Some(ref meta) => match meta.hdr_etag {
            Some(ref etag) => orig_req.set("If-None-Match", etag),
            None => match meta.hdr_last_modified {
                Some(ref last_modified) => orig_req.set("If-Modified-Since", last_modified),
                None => orig_req,
            },
        },
    };
    Ok((req, stored_meta))
}