omni-dev 0.46.0

//! `manifest.jsonl`: the only record of Gmail-side metadata for an archived
//! message.
//!
//! Gmail labels are not part of the RFC 2822 body, so a `labelsAdded`
//! history event cannot be represented by rewriting the `.eml` — label
//! changes update the manifest record only. This is *not* a derived index
//! that could be regenerated from the `.eml` files; it is the sole record
//! of label state, so (unlike `state.rs`'s deliberately disposable
//! `state.json`) a corrupt manifest is a hard failure, never a silent
//! start-from-empty that would orphan every message's label history.

use std::collections::BTreeMap;
use std::io::Write as _;
use std::path::{Path, PathBuf};

use anyhow::{Context, Result};
use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};

/// One archived message's Gmail-side metadata.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub(crate) struct ManifestRecord {
    pub(crate) id: String,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub(crate) thread_id: Option<String>,
    #[serde(default, skip_serializing_if = "Vec::is_empty")]
    pub(crate) label_ids: Vec<String>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub(crate) internal_date: Option<String>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub(crate) subject: Option<String>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub(crate) from: Option<String>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub(crate) to: Option<String>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub(crate) rfc822_msgid: Option<String>,
    /// The `In-Reply-To` header — together with `references`, what lets a
    /// conversation be reconstructed from the manifest alone without
    /// re-parsing every `.eml`.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub(crate) in_reply_to: Option<String>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub(crate) references: Option<String>,
    /// Number of `Content-Disposition: attachment` MIME parts found,
    /// regardless of whether a filename was parseable — see
    /// `crate::gmail::raw_message::extract_attachment_filenames`.
    #[serde(default)]
    pub(crate) attachment_count: u32,
    /// Filenames actually extracted (a subset of `attachment_count`, since
    /// not every attachment part names itself). Always from the same cheap
    /// heuristic scan, regardless of `--extract-attachments` — this field
    /// is independent of whatever that flag wrote to disk this run; see
    /// `crate::gmail::attachments` / ADR-0065.
    #[serde(default, skip_serializing_if = "Vec::is_empty")]
    pub(crate) attachment_filenames: Vec<String>,
    /// Path to the `.eml` file, relative to the archive root.
    pub(crate) path: PathBuf,
    pub(crate) size: u64,
    /// The watermark this record was last touched at.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub(crate) history_id: Option<String>,
    /// Soft-delete marker — set when the message no longer appears in a
    /// full listing. The `.eml` is never removed; a later listing showing
    /// the id again clears this via [`Manifest::undelete`].
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub(crate) deleted_at: Option<DateTime<Utc>>,
    /// The `--exclude-label` value(s) currently known to explain why this
    /// record is soft-deleted (#1780) — always empty when `deleted_at` is
    /// unset. Set only by [`Manifest::mark_excluded`]/
    /// [`Manifest::set_excluded_labels`] (`run_incremental`'s own
    /// labelsAdded/labelsRemoved-driven tracking, the one place the
    /// record's current label set is known precisely rather than guessed);
    /// [`Manifest::mark_deleted`] (a real `messagesDeleted` event, or
    /// `run_full_sync`'s listing-absence pass) always clears it, since a
    /// generic deletion for an unrelated reason must never be mistaken for
    /// a label exclusion later. This precision is load-bearing: it's what
    /// lets an undelete-on-`labelsRemoved` fire safely and automatically,
    /// without risking resurrecting a message soft-deleted for a reason
    /// that has nothing to do with labels.
    #[serde(default, skip_serializing_if = "Vec::is_empty")]
    pub(crate) excluded_labels: Vec<String>,
}

impl ManifestRecord {
    /// Parses [`Self::internal_date`] into a UTC timestamp — mirrors
    /// [`crate::gmail::types::Message::internal_date_utc`] exactly, since a
    /// manifest record's `internal_date` is that same Gmail
    /// epoch-millisecond string, just persisted to disk rather than held on
    /// a freshly-fetched `Message`.
    pub(crate) fn internal_date_utc(&self) -> Option<DateTime<Utc>> {
        let ms: i64 = self.internal_date.as_deref()?.parse().ok()?;
        DateTime::from_timestamp_millis(ms)
    }
}

/// The whole manifest, keyed by message id (a `BTreeMap` for deterministic,
/// greppable output order).
///
/// Loaded entirely into memory, mutated, and rewritten wholesale on
/// [`Self::save`] — not appended-then-compacted. An append-only log can't
/// represent "update this existing record's labels" without a
/// last-write-wins reconstruction on every read; at realistic scale (50k
/// messages × ~250 bytes ≈ 12 MB) loading the whole thing is trivial. The
/// file stays one-JSON-object-per-line; only the *write* strategy is
/// "recompute and replace".
#[derive(Debug, Default)]
pub(crate) struct Manifest(BTreeMap<String, ManifestRecord>);

impl Manifest {
    /// Loads `manifest.jsonl`. An absent file is an empty manifest (first
    /// run); a present-but-unparseable file is a hard error — see the
    /// module doc for why this differs from `state.rs::load`.
    pub(crate) fn load(path: &Path) -> Result<Self> {
        let text = match std::fs::read_to_string(path) {
            Ok(text) => text,
            Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Self::default()),
            Err(e) => {
                return Err(e)
                    .with_context(|| format!("Failed to read manifest at {}", path.display()))
            }
        };
        let mut map = BTreeMap::new();
        for (i, line) in text.lines().enumerate() {
            let line = line.trim();
            if line.is_empty() {
                continue;
            }
            let record: ManifestRecord = serde_json::from_str(line).with_context(|| {
                format!(
                    "Failed to parse manifest line {} at {} — the manifest is the only record \
                     of this archive's label state and is never silently discarded; fix or \
                     restore it from a backup before re-running sync",
                    i + 1,
                    path.display()
                )
            })?;
            map.insert(record.id.clone(), record);
        }
        Ok(Self(map))
    }

    pub(crate) fn get(&self, id: &str) -> Option<&ManifestRecord> {
        self.0.get(id)
    }

    pub(crate) fn upsert(&mut self, record: ManifestRecord) {
        self.0.insert(record.id.clone(), record);
    }

    /// Adds labels to an existing record. A no-op if `id` isn't in the
    /// manifest yet (e.g. its own `messagesAdded` event hasn't been
    /// processed within this run) rather than an error — labels-only
    /// history events never fetch or create a message on their own.
    pub(crate) fn add_labels(&mut self, id: &str, label_ids: &[String]) {
        if let Some(record) = self.0.get_mut(id) {
            for label in label_ids {
                if !record.label_ids.contains(label) {
                    record.label_ids.push(label.clone());
                }
            }
        }
    }

    /// Removes labels from an existing record. A no-op if `id` is absent.
    pub(crate) fn remove_labels(&mut self, id: &str, label_ids: &[String]) {
        if let Some(record) = self.0.get_mut(id) {
            record.label_ids.retain(|l| !label_ids.contains(l));
        }
    }

    /// Soft-deletes a record for a reason unrelated to `--exclude-label`
    /// tracking (a real `messagesDeleted` event, or `run_full_sync`'s
    /// listing-absence pass): the `.eml` is never touched, only
    /// `deleted_at` is set (a no-op if `id` is absent). Also clears any
    /// previously-tracked [`ManifestRecord::excluded_labels`] (#1780) — once
    /// this generic path has touched a record, its `deleted_at` no longer
    /// means what that tracking assumed, so a stale reason must never
    /// survive to justify a later automatic undelete. Use
    /// [`Self::mark_excluded`] instead when the reason *is* a label match.
    pub(crate) fn mark_deleted(&mut self, id: &str, at: DateTime<Utc>) {
        if let Some(record) = self.0.get_mut(id) {
            record.deleted_at = Some(at);
            record.excluded_labels.clear();
        }
    }

    /// Soft-deletes a record specifically because its current label set
    /// matches `--exclude-label` (#1780) — unlike [`Self::mark_deleted`],
    /// this records exactly which configured labels caused it, which is
    /// what lets an undelete on the reverse transition
    /// (`run_incremental`'s `reconcile_label_exclusion`) fire safely and
    /// automatically instead of guessing from possibly-stale cached state.
    pub(crate) fn mark_excluded(
        &mut self,
        id: &str,
        at: DateTime<Utc>,
        excluded_labels: Vec<String>,
    ) {
        if let Some(record) = self.0.get_mut(id) {
            record.deleted_at = Some(at);
            record.excluded_labels = excluded_labels;
        }
    }

    /// Updates the label-exclusion reason [`Self::mark_excluded`] recorded,
    /// without changing `deleted_at` — used when a still-excluded record's
    /// *specific* matching label(s) change (e.g. one of two configured
    /// excluded labels is removed from the message but another still
    /// applies, so it must stay deleted rather than undelete).
    pub(crate) fn set_excluded_labels(&mut self, id: &str, excluded_labels: Vec<String>) {
        if let Some(record) = self.0.get_mut(id) {
            record.excluded_labels = excluded_labels;
        }
    }

    /// Clears a previously-set soft-delete marker (a no-op if `id` is
    /// absent or not currently deleted) — also clears
    /// [`ManifestRecord::excluded_labels`], since an undeleted record isn't
    /// excluded by definition (#1780).
    pub(crate) fn undelete(&mut self, id: &str) {
        if let Some(record) = self.0.get_mut(id) {
            record.deleted_at = None;
            record.excluded_labels.clear();
        }
    }

    /// Ids of every record not currently soft-deleted.
    pub(crate) fn ids_not_deleted(&self) -> impl Iterator<Item = &str> {
        self.0
            .values()
            .filter(|r| r.deleted_at.is_none())
            .map(|r| r.id.as_str())
    }

    /// Every record not currently soft-deleted, with full data — the
    /// `ManifestRecord`-yielding counterpart of [`Self::ids_not_deleted`],
    /// for callers (e.g. `gmail extract-attachments`) that need
    /// `attachment_count`/`path`/`internal_date` rather than just an id to
    /// look up.
    pub(crate) fn records_not_deleted(&self) -> impl Iterator<Item = &ManifestRecord> {
        self.0.values().filter(|r| r.deleted_at.is_none())
    }

    /// Atomically rewrites `manifest.jsonl` in full (temp file in the same
    /// directory + rename via the `tempfile` crate, which additionally
    /// cleans up the temp file if `persist` isn't reached — the same
    /// pattern `src/cli/ai/claude/history/sync.rs` uses for its own atomic
    /// writes).
    pub(crate) fn save(&self, path: &Path) -> Result<()> {
        let dir = path.parent().filter(|p| !p.as_os_str().is_empty());
        let dir = dir.unwrap_or_else(|| Path::new("."));
        let mut tmp = tempfile::NamedTempFile::new_in(dir)
            .with_context(|| format!("Failed to create a temp file in {}", dir.display()))?;
        for record in self.0.values() {
            serde_json::to_writer(&mut tmp, record)
                .context("Failed to serialise a manifest record")?;
            tmp.write_all(b"\n")
                .context("Failed to write a manifest record")?;
        }
        tmp.flush().context("Failed to flush the manifest")?;
        tmp.persist(path)
            .map_err(|e| e.error)
            .with_context(|| format!("Failed to publish manifest to {}", path.display()))?;
        Ok(())
    }
}

#[cfg(test)]
#[allow(clippy::unwrap_used, clippy::expect_used)]
mod tests {
    use super::*;

    fn sample_record(id: &str) -> ManifestRecord {
        ManifestRecord {
            id: id.to_string(),
            thread_id: Some("t1".to_string()),
            label_ids: vec!["INBOX".to_string()],
            internal_date: Some("1700000000000".to_string()),
            subject: Some("Hello".to_string()),
            from: Some("a@example.com".to_string()),
            to: Some("b@example.com".to_string()),
            rfc822_msgid: Some("<1@example.com>".to_string()),
            in_reply_to: None,
            references: None,
            attachment_count: 0,
            attachment_filenames: Vec::new(),
            path: PathBuf::from(format!("messages/2026/01/01/{id}.eml")),
            size: 42,
            history_id: Some("1000".to_string()),
            deleted_at: None,
            excluded_labels: Vec::new(),
        }
    }

    // ── load ─────────────────────────────────────────────────────────

    #[test]
    fn load_absent_file_is_an_empty_manifest() {
        let dir = tempfile::tempdir().unwrap();
        let manifest = Manifest::load(&dir.path().join("manifest.jsonl")).unwrap();
        assert!(manifest.get("m1").is_none());
    }

    #[test]
    fn load_rejects_a_corrupt_line_instead_of_starting_empty() {
        let dir = tempfile::tempdir().unwrap();
        let path = dir.path().join("manifest.jsonl");
        std::fs::write(&path, "{\"id\": \"m1\"\nnot json at all\n").unwrap();
        let err = Manifest::load(&path).unwrap_err();
        assert!(err.to_string().contains("Failed to parse manifest line"));
    }

    #[test]
    fn load_skips_blank_lines() {
        let dir = tempfile::tempdir().unwrap();
        let path = dir.path().join("manifest.jsonl");
        let mut manifest = Manifest::default();
        manifest.upsert(sample_record("m1"));
        manifest.save(&path).unwrap();
        let raw = std::fs::read_to_string(&path).unwrap();
        std::fs::write(&path, format!("\n{raw}\n")).unwrap();

        let loaded = Manifest::load(&path).unwrap();
        assert!(loaded.get("m1").is_some());
    }

    // ── round-trip / upsert ────────────────────────────────────────────

    #[test]
    fn save_then_load_round_trips_a_record() {
        let dir = tempfile::tempdir().unwrap();
        let path = dir.path().join("manifest.jsonl");
        let mut manifest = Manifest::default();
        manifest.upsert(sample_record("m1"));
        manifest.save(&path).unwrap();

        let loaded = Manifest::load(&path).unwrap();
        assert_eq!(loaded.get("m1"), Some(&sample_record("m1")));
    }

    #[test]
    fn save_writes_one_compact_json_line_per_record() {
        let dir = tempfile::tempdir().unwrap();
        let path = dir.path().join("manifest.jsonl");
        let mut manifest = Manifest::default();
        manifest.upsert(sample_record("m1"));
        manifest.upsert(sample_record("m2"));
        manifest.save(&path).unwrap();

        let text = std::fs::read_to_string(&path).unwrap();
        assert_eq!(text.lines().count(), 2);
    }

    #[test]
    fn save_is_atomic_and_leaves_no_temp_file_behind() {
        let dir = tempfile::tempdir().unwrap();
        let path = dir.path().join("manifest.jsonl");
        let mut manifest = Manifest::default();
        manifest.upsert(sample_record("m1"));
        manifest.save(&path).unwrap();

        let leftover: Vec<_> = std::fs::read_dir(dir.path())
            .unwrap()
            .filter_map(std::result::Result::ok)
            .filter(|e| e.path() != path)
            .collect();
        assert!(leftover.is_empty(), "expected no leftover temp files");
    }

    // ── labels ───────────────────────────────────────────────────────

    #[test]
    fn add_labels_appends_without_duplicating() {
        let mut manifest = Manifest::default();
        manifest.upsert(sample_record("m1"));
        manifest.add_labels("m1", &["IMPORTANT".to_string(), "INBOX".to_string()]);
        assert_eq!(
            manifest.get("m1").unwrap().label_ids,
            vec!["INBOX".to_string(), "IMPORTANT".to_string()]
        );
    }

    #[test]
    fn add_labels_is_a_no_op_when_id_is_absent() {
        let mut manifest = Manifest::default();
        manifest.add_labels("missing", &["INBOX".to_string()]);
        assert!(manifest.get("missing").is_none());
    }

    #[test]
    fn remove_labels_drops_only_the_named_labels() {
        let mut manifest = Manifest::default();
        let mut record = sample_record("m1");
        record.label_ids = vec!["INBOX".to_string(), "UNREAD".to_string()];
        manifest.upsert(record);
        manifest.remove_labels("m1", &["UNREAD".to_string()]);
        assert_eq!(
            manifest.get("m1").unwrap().label_ids,
            vec!["INBOX".to_string()]
        );
    }

    // ── soft-delete ────────────────────────────────────────────────────

    #[test]
    fn mark_deleted_sets_deleted_at_and_undelete_clears_it() {
        let mut manifest = Manifest::default();
        manifest.upsert(sample_record("m1"));
        let now = Utc::now();
        manifest.mark_deleted("m1", now);
        assert_eq!(manifest.get("m1").unwrap().deleted_at, Some(now));

        manifest.undelete("m1");
        assert_eq!(manifest.get("m1").unwrap().deleted_at, None);
    }

    #[test]
    fn mark_deleted_clears_any_previously_tracked_excluded_labels() {
        let mut manifest = Manifest::default();
        manifest.upsert(sample_record("m1"));
        manifest.mark_excluded("m1", Utc::now(), vec!["SPAM".to_string()]);
        assert_eq!(manifest.get("m1").unwrap().excluded_labels, vec!["SPAM"]);

        // A generic deletion (e.g. a real messagesDeleted event) must not
        // leave a stale exclusion reason behind for a later undelete to
        // trust (#1780).
        manifest.mark_deleted("m1", Utc::now());
        assert!(manifest.get("m1").unwrap().excluded_labels.is_empty());
    }

    #[test]
    fn mark_excluded_sets_deleted_at_and_the_exact_reason() {
        let mut manifest = Manifest::default();
        manifest.upsert(sample_record("m1"));
        let now = Utc::now();
        manifest.mark_excluded("m1", now, vec!["SPAM".to_string(), "TRASH".to_string()]);
        let record = manifest.get("m1").unwrap();
        assert_eq!(record.deleted_at, Some(now));
        assert_eq!(record.excluded_labels, vec!["SPAM", "TRASH"]);
    }

    #[test]
    fn set_excluded_labels_updates_the_reason_without_touching_deleted_at() {
        let mut manifest = Manifest::default();
        manifest.upsert(sample_record("m1"));
        let now = Utc::now();
        manifest.mark_excluded("m1", now, vec!["SPAM".to_string(), "TRASH".to_string()]);

        manifest.set_excluded_labels("m1", vec!["SPAM".to_string()]);
        let record = manifest.get("m1").unwrap();
        assert_eq!(record.deleted_at, Some(now), "deleted_at must be untouched");
        assert_eq!(record.excluded_labels, vec!["SPAM"]);
    }

    #[test]
    fn undelete_clears_excluded_labels() {
        let mut manifest = Manifest::default();
        manifest.upsert(sample_record("m1"));
        manifest.mark_excluded("m1", Utc::now(), vec!["SPAM".to_string()]);

        manifest.undelete("m1");
        let record = manifest.get("m1").unwrap();
        assert_eq!(record.deleted_at, None);
        assert!(record.excluded_labels.is_empty());
    }

    #[test]
    fn ids_not_deleted_excludes_soft_deleted_records() {
        let mut manifest = Manifest::default();
        manifest.upsert(sample_record("m1"));
        manifest.upsert(sample_record("m2"));
        manifest.mark_deleted("m1", Utc::now());

        let remaining: Vec<&str> = manifest.ids_not_deleted().collect();
        assert_eq!(remaining, vec!["m2"]);
    }

    #[test]
    fn records_not_deleted_excludes_soft_deleted_records() {
        let mut manifest = Manifest::default();
        manifest.upsert(sample_record("m1"));
        manifest.upsert(sample_record("m2"));
        manifest.mark_deleted("m1", Utc::now());

        let remaining: Vec<&str> = manifest
            .records_not_deleted()
            .map(|r| r.id.as_str())
            .collect();
        assert_eq!(remaining, vec!["m2"]);
    }

    // ── internal_date_utc ────────────────────────────────────────────

    #[test]
    fn internal_date_utc_parses_valid_epoch_ms() {
        let record = sample_record("m1");
        assert_eq!(
            record.internal_date_utc(),
            DateTime::from_timestamp_millis(1_700_000_000_000)
        );
    }

    #[test]
    fn internal_date_utc_is_none_for_missing_or_invalid() {
        let mut record = sample_record("m1");
        record.internal_date = None;
        assert_eq!(record.internal_date_utc(), None);

        record.internal_date = Some("not a number".to_string());
        assert_eq!(record.internal_date_utc(), None);
    }

    #[test]
    fn label_only_change_never_touches_the_eml_path_or_size() {
        let mut manifest = Manifest::default();
        manifest.upsert(sample_record("m1"));
        let before = manifest.get("m1").unwrap().clone();
        manifest.add_labels("m1", &["IMPORTANT".to_string()]);
        let after = manifest.get("m1").unwrap();
        assert_eq!(before.path, after.path);
        assert_eq!(before.size, after.size);
    }
}