arcform 0.2.2

//! The ingress **freshness contract** — content identity persisted next to a
//! fetched artifact.
//!
//! An `http_fetch` writes a sidecar (`<out>.arcmeta`, YAML) recording the remote
//! identity of what it pulled: the `ETag`/`Last-Modified` the server returned, the
//! `sha256` of the bytes, and — where the origin publishes one ahead of the transfer
//! — the content hash it declared. That sidecar is what makes a fetch *content-aware*
//! rather than clock-based:
//!   - the operator replays the stored `ETag`/`Last-Modified` as an `If-None-Match`
//!     / `If-Modified-Since` conditional request, so an unchanged remote returns
//!     `304` and the bytes are **not** re-downloaded. The stored validators are the
//!     **first** hop's where that hop sends one — the hop `url` addresses — and the
//!     last hop's otherwise, with the conditional forwarded down the chain;
//!   - the [`crate::precondition`] `fresh` gate hashes the artifact against the
//!     recorded `sha256` and, when that agrees, HEAD-probes the same identity — so a
//!     step (and everything downstream) re-runs when the local bytes or the remote
//!     moved, the content-addressed replacement for the mtime `modified_after` gate.
//!
//! Because the stored validators are what make the next request conditional, the
//! sidecar is also the lever for *forcing* a full re-download: [`remove`] it and the
//! operator has nothing to replay, so the server answers with bytes rather than a
//! `304`. The `fresh` gate does exactly that when the artifact fails its hash.
//!
//! Written by `http_fetch`, read by the `fresh` precondition — hence its own module.

use std::collections::BTreeMap;
use std::path::{Path, PathBuf};

use serde::{Deserialize, Serialize};

/// Default User-Agent for ingress requests. An unset UA is a `403` at the SEC and
/// several registries — shared by the operator's fetch and the `fresh` probe so the
/// conditional HEAD hits the same endpoint the fetch did.
pub const DEFAULT_UA: &str = "arcform-http_fetch/1 (+https://meridian.online)";

/// Remote identity of a fetched artifact, persisted as `<out>.arcmeta`.
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct FetchMeta {
    /// The URL fetched — the `fresh` probe re-requests exactly this.
    pub url: String,
    /// Request headers the fetch sent (e.g. a UA) — replayed by the probe so a
    /// header-gated source is probed the same way it was fetched.
    #[serde(default)]
    pub request_headers: BTreeMap<String, String>,
    /// The server's `ETag`, if any — the primary conditional/freshness key.
    #[serde(default)]
    pub etag: Option<String>,
    /// The server's `Last-Modified`, if any — the fallback key.
    #[serde(default)]
    pub last_modified: Option<String>,
    /// SHA-256 of the fetched bytes — the content identity (audit + reproducibility).
    #[serde(default)]
    pub sha256: String,
    /// The content hash the origin declared for the artifact, taken from the response
    /// head rather than from the body. Skipped on write when absent, so a sidecar for
    /// an origin that declares none is byte-identical to one written before this
    /// field existed.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub content_sha256: Option<String>,
    /// Best-effort fetch time (unix seconds) — audit only.
    #[serde(default)]
    pub fetched_unix: Option<u64>,
}

/// Sidecar path for a fetched artifact: `<out>.arcmeta` (appended, so
/// `build/edgar.parquet` → `build/edgar.parquet.arcmeta`).
pub fn meta_path(out: &Path) -> PathBuf {
    let mut s = out.as_os_str().to_owned();
    s.push(".arcmeta");
    PathBuf::from(s)
}

/// Read the sidecar for `out`, or `None` if it's absent or unparseable.
pub fn read(out: &Path) -> Option<FetchMeta> {
    let s = std::fs::read_to_string(meta_path(out)).ok()?;
    serde_yaml::from_str(&s).ok()
}

/// Write the sidecar for `out`.
pub fn write(out: &Path, meta: &FetchMeta) -> std::io::Result<()> {
    let s = serde_yaml::to_string(meta).map_err(std::io::Error::other)?;
    std::fs::write(meta_path(out), s)
}

/// Delete the sidecar for `out`, leaving the artifact itself alone. An already-absent
/// sidecar is `Ok(())`, so calling this on an artifact that was never fetched is not an
/// error.
///
/// The next `http_fetch` of `out` then finds no `ETag`/`Last-Modified` to replay and
/// issues an unconditional `GET`.
pub fn remove(out: &Path) -> std::io::Result<()> {
    match std::fs::remove_file(meta_path(out)) {
        Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),
        other => other,
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn meta_path_appends_not_replaces() {
        // `build/edgar.parquet` → `build/edgar.parquet.arcmeta` (keeps the extension).
        assert_eq!(
            meta_path(Path::new("build/edgar.parquet")),
            PathBuf::from("build/edgar.parquet.arcmeta")
        );
    }

    #[test]
    fn sidecar_round_trips() {
        let dir = tempfile::tempdir().unwrap();
        let out = dir.path().join("edgar.parquet");
        let mut headers = BTreeMap::new();
        headers.insert("User-Agent".to_string(), "meridian".to_string());
        let meta = FetchMeta {
            url: "https://openlake.meridian.online/edgar.parquet".to_string(),
            request_headers: headers,
            etag: Some("\"abc123\"".to_string()),
            last_modified: Some("Sat, 04 Jul 2026 06:31:19 GMT".to_string()),
            sha256: "deadbeef".to_string(),
            content_sha256: Some("c0ffee".to_string()),
            fetched_unix: Some(1_784_000_000),
        };
        write(&out, &meta).unwrap();
        let back = read(&out).expect("sidecar reads back");
        assert_eq!(back.url, meta.url);
        assert_eq!(back.etag, meta.etag);
        assert_eq!(back.last_modified, meta.last_modified);
        assert_eq!(back.sha256, meta.sha256);
        assert_eq!(back.content_sha256, meta.content_sha256);
        assert_eq!(
            back.request_headers.get("User-Agent").map(String::as_str),
            Some("meridian")
        );
    }

    // An origin that declares no content hash writes the same bytes it wrote before
    // the field existed, so an existing sidecar is not rewritten by upgrading.
    #[test]
    fn absent_content_hash_is_not_serialised() {
        let dir = tempfile::tempdir().unwrap();
        let out = dir.path().join("edgar.parquet");
        let meta = FetchMeta {
            url: "https://openlake.meridian.online/edgar.parquet".to_string(),
            etag: Some("\"abc123\"".to_string()),
            sha256: "deadbeef".to_string(),
            ..Default::default()
        };
        write(&out, &meta).unwrap();
        let on_disk = std::fs::read_to_string(meta_path(&out)).unwrap();
        assert!(!on_disk.contains("content_sha256"), "{}", on_disk);
        assert!(read(&out).unwrap().content_sha256.is_none());
    }

    #[test]
    fn read_absent_sidecar_is_none() {
        let dir = tempfile::tempdir().unwrap();
        assert!(read(&dir.path().join("nope.parquet")).is_none());
    }

    #[test]
    fn remove_deletes_the_sidecar_and_leaves_the_artifact() {
        let dir = tempfile::tempdir().unwrap();
        let out = dir.path().join("edgar.parquet");
        std::fs::write(&out, b"bytes").unwrap();
        write(&out, &FetchMeta::default()).unwrap();

        remove(&out).unwrap();
        assert!(!meta_path(&out).exists(), "sidecar gone");
        assert!(out.is_file(), "artifact kept");
        assert!(read(&out).is_none());
    }

    #[test]
    fn remove_absent_sidecar_is_ok() {
        let dir = tempfile::tempdir().unwrap();
        remove(&dir.path().join("never-fetched.parquet")).unwrap();
    }
}