use std::io::IsTerminal as _;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use anyhow::{Context, Result};
use clap::Parser;
use futures::stream::{FuturesUnordered, StreamExt as _};
use serde::{Deserialize, Serialize};
use tokio::sync::{mpsc, Semaphore};
use crate::claude::context::discovery;
use crate::gmail::account::validate_account;
use crate::gmail::client::GmailClient;
use crate::utils::settings::Settings;
use super::format::{output_as, write_scalar_jsonl, JsonlSerialize, OutputFormat};
use super::helpers;
use super::sync::engine::{self, SyncOptions};
use super::sync::progress::{SyncProgressBars, SyncProgressEvent};
use super::sync::report::{SyncAction, SyncError, SyncReport, SyncSummary};
use super::sync::{format_summary_line, should_show_progress, DEFAULT_SYNC_CONCURRENCY};
#[derive(Debug, Default, Deserialize)]
pub(crate) struct GmailSyncAllConfig {
#[serde(default)]
pub(crate) concurrency: Option<usize>,
#[serde(default)]
pub(crate) accounts: Vec<GmailSyncAccountEntry>,
}
#[derive(Debug, Deserialize)]
pub(crate) struct GmailSyncAccountEntry {
pub(crate) account: String,
pub(crate) output_dir: PathBuf,
#[serde(default)]
pub(crate) query: Option<String>,
#[serde(default)]
pub(crate) extract_attachments: Option<bool>,
}
pub(crate) fn load_gmail_sync_config(context_dir: &Path) -> Result<GmailSyncAllConfig> {
let path = discovery::resolve_config_file(context_dir, "gmail-sync.yaml");
if !path.exists() {
anyhow::bail!(
"no gmail-sync.yaml found (looked under {}); add one with an `accounts:` list, or \
point --context-dir/OMNI_DEV_CONFIG_DIR at a directory containing one",
context_dir.display()
);
}
let raw = std::fs::read_to_string(&path)
.with_context(|| format!("Failed to read {}", path.display()))?;
let config: GmailSyncAllConfig = serde_yaml::from_str(&raw)
.with_context(|| format!("Failed to parse {}", path.display()))?;
if config.accounts.is_empty() {
anyhow::bail!(
"{} has no accounts configured; add entries to its `accounts:` list",
path.display()
);
}
Ok(config)
}
fn validate_accounts(settings: &Settings, accounts: &[GmailSyncAccountEntry]) -> Result<()> {
let unknown: Vec<&str> = accounts
.iter()
.filter(|entry| validate_account(&settings.gmail, &entry.account).is_err())
.map(|entry| entry.account.as_str())
.collect();
if unknown.is_empty() {
return Ok(());
}
anyhow::bail!(
"gmail-sync.yaml references unknown Gmail account(s): {}. Run `gmail account list` to \
see configured accounts.",
unknown.join(", ")
);
}
#[derive(Parser)]
pub struct SyncAllCommand {
#[arg(long, value_name = "PATH")]
pub context_dir: Option<PathBuf>,
#[arg(long, value_name = "N")]
pub concurrency: Option<usize>,
#[arg(long)]
pub full: bool,
#[arg(long)]
pub dry_run: bool,
#[arg(long)]
pub quiet: bool,
#[arg(short = 'o', long, value_enum, default_value_t = OutputFormat::Table)]
pub output: OutputFormat,
}
#[derive(Serialize)]
struct AccountSyncOutcome {
account: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
account_error: Option<String>,
actions: Vec<SyncAction>,
errors: Vec<SyncError>,
summary: SyncSummary,
}
#[derive(Serialize)]
struct SyncAllReportOutput {
accounts: Vec<AccountSyncOutcome>,
combined_summary: SyncSummary,
}
impl JsonlSerialize for SyncAllReportOutput {
fn write_jsonl(&self, out: &mut dyn std::io::Write) -> Result<()> {
write_scalar_jsonl(self, out)
}
}
type ClientFor = Arc<dyn Fn(&str) -> Result<GmailClient> + Send + Sync>;
impl SyncAllCommand {
pub(crate) async fn execute(self) -> Result<()> {
let (context_dir, _source) =
discovery::resolve_context_dir_with_source(self.context_dir.as_deref());
let config = load_gmail_sync_config(&context_dir)?;
let settings = Settings::load().context("Failed to load ~/.omni-dev/settings.json")?;
validate_accounts(&settings, &config.accounts)?;
let effective_concurrency = self
.concurrency
.or(config.concurrency)
.unwrap_or(DEFAULT_SYNC_CONCURRENCY)
.max(1);
let project_root: PathBuf = context_dir
.parent()
.map_or_else(|| context_dir.clone(), Path::to_path_buf);
let client_for: ClientFor =
Arc::new(|account: &str| helpers::create_client_for(Some(account)));
run_sync_all(
&config,
&project_root,
self.full,
self.dry_run,
self.quiet,
&self.output,
effective_concurrency,
client_for,
)
.await
}
}
#[allow(clippy::too_many_arguments)]
async fn run_sync_all(
config: &GmailSyncAllConfig,
project_root: &Path,
full: bool,
dry_run: bool,
quiet: bool,
output: &OutputFormat,
effective_concurrency: usize,
client_for: ClientFor,
) -> Result<()> {
let pool = Arc::new(Semaphore::new(effective_concurrency));
let show_progress = should_show_progress(quiet, output, std::io::stderr().is_terminal());
let multi = show_progress.then(indicatif::MultiProgress::new);
let mut render_tasks = Vec::new();
let mut tasks = FuturesUnordered::new();
for entry in &config.accounts {
let account = entry.account.clone();
let output_dir = if entry.output_dir.is_relative() {
project_root.join(&entry.output_dir)
} else {
entry.output_dir.clone()
};
let opts = SyncOptions {
output_dir,
query: entry.query.clone(),
full,
concurrency: DEFAULT_SYNC_CONCURRENCY,
dry_run,
extract_attachments: entry.extract_attachments.unwrap_or(false),
shared_pool: Some(Arc::clone(&pool)),
};
let progress_tx = multi.as_ref().map(|multi| {
let bars = SyncProgressBars::new_in(multi, &account);
let (tx, rx) = mpsc::unbounded_channel();
render_tasks.push(tokio::spawn(bars.drain(rx)));
tx
});
let spawn_account = account.clone();
let spawn_client_for = Arc::clone(&client_for);
let join = tokio::spawn(async move {
run_one_account(
&spawn_account,
&opts,
spawn_client_for.as_ref(),
progress_tx.as_ref(),
)
.await
});
tasks.push(async move {
let outcome = match join.await {
Ok(result) => result,
Err(join_err) => Err(anyhow::anyhow!("sync task panicked: {join_err}")),
};
(account, outcome)
});
}
let show_text = matches!(output, OutputFormat::Table);
let mut outcomes: Vec<(String, Result<SyncReport>)> = Vec::with_capacity(config.accounts.len());
while let Some((account, outcome)) = tasks.next().await {
if show_text {
match &multi {
Some(multi) => multi.suspend(|| print_account_line(&account, &outcome, quiet)),
None => print_account_line(&account, &outcome, quiet),
}
}
outcomes.push((account, outcome));
}
for render_task in render_tasks {
let _ = render_task.await;
}
let mut combined = SyncSummary::default();
let mut any_failed = false;
let mut accounts_output = Vec::with_capacity(outcomes.len());
for (account, outcome) in outcomes {
match outcome {
Ok(report) => {
let summary = report.summary();
if !report.errors.is_empty() {
any_failed = true;
}
add_summary(&mut combined, &summary);
accounts_output.push(AccountSyncOutcome {
account,
account_error: None,
actions: report.actions,
errors: report.errors,
summary,
});
}
Err(err) => {
any_failed = true;
combined.errors += 1;
accounts_output.push(AccountSyncOutcome {
account,
account_error: Some(format!("{err:#}")),
actions: Vec::new(),
errors: Vec::new(),
summary: SyncSummary {
errors: 1,
..SyncSummary::default()
},
});
}
}
}
if show_text {
println!("combined: {}", format_summary_line(&combined));
} else {
output_as(
&SyncAllReportOutput {
accounts: accounts_output,
combined_summary: combined,
},
output,
)?;
}
if any_failed {
anyhow::bail!("one or more accounts failed to sync; see output above");
}
Ok(())
}
async fn run_one_account(
account: &str,
opts: &SyncOptions,
client_for: &(dyn Fn(&str) -> Result<GmailClient> + Send + Sync),
progress: Option<&mpsc::UnboundedSender<SyncProgressEvent>>,
) -> Result<SyncReport> {
let client = client_for(account)
.with_context(|| format!("account '{account}': failed to build a Gmail client"))?;
engine::run_sync_with_progress(&client, opts, progress)
.await
.with_context(|| format!("account '{account}': sync failed"))
}
fn print_account_line(account: &str, outcome: &Result<SyncReport>, quiet: bool) {
match outcome {
Ok(report) => {
for error in &report.errors {
println!("{account}: error: {} failed: {}", error.id, error.reason);
}
if !quiet {
println!("{account}: {}", format_summary_line(&report.summary()));
}
}
Err(err) => println!("{account}: failed: {err:#}"),
}
}
fn add_summary(total: &mut SyncSummary, summary: &SyncSummary) {
total.fetched += summary.fetched;
total.would_fetch += summary.would_fetch;
total.vanished += summary.vanished;
total.labels_updated += summary.labels_updated;
total.deleted += summary.deleted;
total.undeleted += summary.undeleted;
total.would_delete += summary.would_delete;
total.would_undelete += summary.would_undelete;
total.errors += summary.errors;
}
#[cfg(test)]
#[allow(clippy::unwrap_used, clippy::expect_used)]
mod tests {
use std::collections::HashMap;
use std::sync::Mutex;
use base64::Engine as _;
use crate::gmail::auth::{GmailCredentials, GmailScope};
use crate::utils::secret::Secret;
use crate::utils::settings::GmailAccountSettings;
use super::*;
#[test]
fn load_gmail_sync_config_parses_accounts_and_concurrency() {
let dir = tempfile::tempdir().unwrap();
std::fs::write(
dir.path().join("gmail-sync.yaml"),
r#"
concurrency: 5
accounts:
- account: jky.greens
output_dir: emails/jky.greens/
- account: newhoggy
output_dir: emails/newhoggy/
query: "-in:spam"
extract_attachments: true
"#,
)
.unwrap();
let config = load_gmail_sync_config(dir.path()).unwrap();
assert_eq!(config.concurrency, Some(5));
assert_eq!(config.accounts.len(), 2);
assert_eq!(config.accounts[0].account, "jky.greens");
assert_eq!(config.accounts[1].query.as_deref(), Some("-in:spam"));
assert_eq!(config.accounts[1].extract_attachments, Some(true));
}
#[test]
fn load_gmail_sync_config_missing_file_errors() {
let dir = tempfile::tempdir().unwrap();
let err = load_gmail_sync_config(dir.path()).unwrap_err();
assert!(err.to_string().contains("no gmail-sync.yaml found"));
}
#[test]
fn load_gmail_sync_config_empty_accounts_errors() {
let dir = tempfile::tempdir().unwrap();
std::fs::write(dir.path().join("gmail-sync.yaml"), "accounts: []\n").unwrap();
let err = load_gmail_sync_config(dir.path()).unwrap_err();
assert!(err.to_string().contains("no accounts configured"));
}
#[test]
fn load_gmail_sync_config_malformed_yaml_errors() {
let dir = tempfile::tempdir().unwrap();
std::fs::write(dir.path().join("gmail-sync.yaml"), "accounts: [not valid").unwrap();
assert!(load_gmail_sync_config(dir.path()).is_err());
}
#[test]
fn load_gmail_sync_config_prefers_local_shadow_file() {
let dir = tempfile::tempdir().unwrap();
std::fs::write(
dir.path().join("gmail-sync.yaml"),
"accounts:\n - account: shared\n output_dir: emails/shared/\n",
)
.unwrap();
std::fs::create_dir_all(dir.path().join("local")).unwrap();
std::fs::write(
dir.path().join("local").join("gmail-sync.yaml"),
"accounts:\n - account: personal\n output_dir: emails/personal/\n",
)
.unwrap();
let config = load_gmail_sync_config(dir.path()).unwrap();
assert_eq!(config.accounts[0].account, "personal");
}
fn account_entry(account: &str) -> GmailSyncAccountEntry {
GmailSyncAccountEntry {
account: account.to_string(),
output_dir: PathBuf::from("out"),
query: None,
extract_attachments: None,
}
}
#[test]
fn validate_accounts_batches_every_unknown_name() {
let mut settings = Settings::default();
settings
.gmail
.accounts
.insert("known".to_string(), GmailAccountSettings::default());
let entries = vec![
account_entry("known"),
account_entry("missing1"),
account_entry("missing2"),
];
let err = validate_accounts(&settings, &entries).unwrap_err();
let message = err.to_string();
assert!(message.contains("missing1"));
assert!(message.contains("missing2"));
assert!(!message.contains("'known'"));
}
#[test]
fn validate_accounts_ok_when_all_known() {
let mut settings = Settings::default();
settings
.gmail
.accounts
.insert("known".to_string(), GmailAccountSettings::default());
assert!(validate_accounts(&settings, &[account_entry("known")]).is_ok());
}
fn test_credentials() -> GmailCredentials {
GmailCredentials {
client_id: "client-1".to_string(),
client_secret: Secret::new("secret-1"),
refresh_token: Secret::new("refresh-1"),
scope: GmailScope::ReadOnly,
}
}
async fn mount_token(server: &wiremock::MockServer) {
wiremock::Mock::given(wiremock::matchers::method("POST"))
.and(wiremock::matchers::path("/token"))
.respond_with(
wiremock::ResponseTemplate::new(200).set_body_json(serde_json::json!({
"access_token": "test-token",
"expires_in": 3600,
})),
)
.mount(server)
.await;
}
fn bootstrapped_client(server: &wiremock::MockServer) -> GmailClient {
let mut client = GmailClient::new(&server.uri(), &test_credentials()).unwrap();
crate::gmail::client::test_support::replace_session(
&mut client,
&test_credentials(),
&format!("{}/token", server.uri()),
);
client
}
async fn mount_profile(server: &wiremock::MockServer) {
wiremock::Mock::given(wiremock::matchers::method("GET"))
.and(wiremock::matchers::path("/gmail/v1/users/me/profile"))
.respond_with(
wiremock::ResponseTemplate::new(200).set_body_json(serde_json::json!({
"emailAddress": "user@example.com",
"messagesTotal": 1,
"threadsTotal": 1,
"historyId": "500",
})),
)
.mount(server)
.await;
}
async fn mount_message_list(server: &wiremock::MockServer, ids: &[&str]) {
let messages: Vec<serde_json::Value> = ids
.iter()
.map(|id| serde_json::json!({"id": id, "threadId": "t1"}))
.collect();
wiremock::Mock::given(wiremock::matchers::method("GET"))
.and(wiremock::matchers::path("/gmail/v1/users/me/messages"))
.respond_with(
wiremock::ResponseTemplate::new(200)
.set_body_json(serde_json::json!({"messages": messages})),
)
.mount(server)
.await;
}
async fn mount_raw_get(server: &wiremock::MockServer, id: &str) {
let source = format!("Subject: Hello\r\nFrom: a@example.com\r\n\r\nBody of {id}.");
let raw = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(source);
wiremock::Mock::given(wiremock::matchers::method("GET"))
.and(wiremock::matchers::path(format!(
"/gmail/v1/users/me/messages/{id}"
)))
.and(wiremock::matchers::query_param("format", "raw"))
.respond_with(
wiremock::ResponseTemplate::new(200).set_body_json(serde_json::json!({
"id": id, "threadId": "t1", "labelIds": ["INBOX"],
"internalDate": "1700000000000", "historyId": "500",
"raw": raw,
})),
)
.mount(server)
.await;
}
#[tokio::test]
async fn run_sync_all_syncs_every_configured_account_concurrently() {
let server = wiremock::MockServer::start().await;
mount_token(&server).await;
mount_profile(&server).await;
mount_message_list(&server, &["m1", "m2"]).await;
mount_raw_get(&server, "m1").await;
mount_raw_get(&server, "m2").await;
let client_a = bootstrapped_client(&server);
let client_b = bootstrapped_client(&server);
let dir = tempfile::tempdir().unwrap();
let project_root = dir.path();
let output_a = project_root.join("emails/acct-a");
let output_b = project_root.join("emails/acct-b");
let config = GmailSyncAllConfig {
concurrency: None,
accounts: vec![
GmailSyncAccountEntry {
account: "acct-a".to_string(),
output_dir: output_a.clone(),
query: None,
extract_attachments: None,
},
GmailSyncAccountEntry {
account: "acct-b".to_string(),
output_dir: output_b.clone(),
query: None,
extract_attachments: None,
},
],
};
let clients = Mutex::new(HashMap::from([
("acct-a".to_string(), client_a),
("acct-b".to_string(), client_b),
]));
let clients = std::sync::Arc::new(clients);
let client_for: ClientFor = std::sync::Arc::new(move |account: &str| {
clients
.lock()
.unwrap()
.remove(account)
.ok_or_else(|| anyhow::anyhow!("no test client for {account}"))
});
run_sync_all(
&config,
project_root,
false,
false,
true,
&OutputFormat::Table,
20,
client_for,
)
.await
.unwrap();
for output_dir in [&output_a, &output_b] {
let manifest = std::fs::read_to_string(output_dir.join("manifest.jsonl")).unwrap();
assert_eq!(
manifest.lines().count(),
2,
"{} should have archived both messages",
output_dir.display()
);
}
}
#[tokio::test]
async fn run_sync_all_fails_overall_but_still_syncs_the_other_account() {
let server = wiremock::MockServer::start().await;
mount_token(&server).await;
mount_profile(&server).await;
mount_message_list(&server, &["m1"]).await;
mount_raw_get(&server, "m1").await;
let client_good = bootstrapped_client(&server);
let dir = tempfile::tempdir().unwrap();
let project_root = dir.path();
let output_good = project_root.join("good");
let config = GmailSyncAllConfig {
concurrency: None,
accounts: vec![
GmailSyncAccountEntry {
account: "good".to_string(),
output_dir: output_good.clone(),
query: None,
extract_attachments: None,
},
GmailSyncAccountEntry {
account: "broken".to_string(),
output_dir: project_root.join("broken"),
query: None,
extract_attachments: None,
},
],
};
let client_good = Mutex::new(Some(client_good));
let client_for: ClientFor = std::sync::Arc::new(move |account: &str| {
if account == "good" {
client_good
.lock()
.unwrap()
.take()
.ok_or_else(|| anyhow::anyhow!("client used twice"))
} else {
Err(anyhow::anyhow!(
"simulated credential failure for {account}"
))
}
});
let err = run_sync_all(
&config,
project_root,
false,
false,
true,
&OutputFormat::Table,
20,
client_for,
)
.await
.unwrap_err();
assert!(err.to_string().contains("one or more accounts failed"));
let manifest = std::fs::read_to_string(output_good.join("manifest.jsonl")).unwrap();
assert_eq!(manifest.lines().count(), 1);
}
#[tokio::test]
async fn run_sync_all_survives_a_spawned_task_panic_and_still_syncs_the_other_account() {
let server = wiremock::MockServer::start().await;
mount_token(&server).await;
mount_profile(&server).await;
mount_message_list(&server, &["m1"]).await;
mount_raw_get(&server, "m1").await;
let client_good = bootstrapped_client(&server);
let dir = tempfile::tempdir().unwrap();
let project_root = dir.path();
let output_good = project_root.join("good");
let config = GmailSyncAllConfig {
concurrency: None,
accounts: vec![
GmailSyncAccountEntry {
account: "good".to_string(),
output_dir: output_good.clone(),
query: None,
extract_attachments: None,
},
GmailSyncAccountEntry {
account: "boom".to_string(),
output_dir: project_root.join("boom"),
query: None,
extract_attachments: None,
},
],
};
let client_good = Mutex::new(Some(client_good));
let client_for: ClientFor = std::sync::Arc::new(move |account: &str| {
assert!(
account != "boom",
"client_for panicked on purpose for {account}"
);
client_good
.lock()
.unwrap()
.take()
.ok_or_else(|| anyhow::anyhow!("client used twice"))
});
let err = run_sync_all(
&config,
project_root,
false,
false,
true,
&OutputFormat::Table,
20,
client_for,
)
.await
.unwrap_err();
assert!(err.to_string().contains("one or more accounts failed"));
let manifest = std::fs::read_to_string(output_good.join("manifest.jsonl")).unwrap();
assert_eq!(manifest.lines().count(), 1);
}
#[tokio::test]
async fn run_sync_all_resolves_relative_output_dir_against_project_root() {
let server = wiremock::MockServer::start().await;
mount_token(&server).await;
mount_profile(&server).await;
mount_message_list(&server, &["m1"]).await;
mount_raw_get(&server, "m1").await;
let client = bootstrapped_client(&server);
let dir = tempfile::tempdir().unwrap();
let project_root = dir.path();
let relative = PathBuf::from("emails/acct-a");
let config = GmailSyncAllConfig {
concurrency: None,
accounts: vec![GmailSyncAccountEntry {
account: "acct-a".to_string(),
output_dir: relative.clone(),
query: None,
extract_attachments: None,
}],
};
let client = Mutex::new(Some(client));
let client_for: ClientFor = std::sync::Arc::new(move |_: &str| {
client
.lock()
.unwrap()
.take()
.ok_or_else(|| anyhow::anyhow!("client used twice"))
});
run_sync_all(
&config,
project_root,
false,
false,
true,
&OutputFormat::Table,
20,
client_for,
)
.await
.unwrap();
let manifest =
std::fs::read_to_string(project_root.join(&relative).join("manifest.jsonl")).unwrap();
assert_eq!(manifest.lines().count(), 1);
}
#[tokio::test]
async fn run_sync_all_writes_structured_output_for_a_non_table_format() {
let server = wiremock::MockServer::start().await;
mount_token(&server).await;
mount_profile(&server).await;
mount_message_list(&server, &["m1"]).await;
mount_raw_get(&server, "m1").await;
let client = bootstrapped_client(&server);
let dir = tempfile::tempdir().unwrap();
let project_root = dir.path();
let output_dir = project_root.join("emails/acct-a");
let config = GmailSyncAllConfig {
concurrency: None,
accounts: vec![GmailSyncAccountEntry {
account: "acct-a".to_string(),
output_dir: output_dir.clone(),
query: None,
extract_attachments: None,
}],
};
let client = Mutex::new(Some(client));
let client_for: ClientFor = std::sync::Arc::new(move |_: &str| {
client
.lock()
.unwrap()
.take()
.ok_or_else(|| anyhow::anyhow!("client used twice"))
});
run_sync_all(
&config,
project_root,
false,
false,
true,
&OutputFormat::Json,
20,
client_for,
)
.await
.unwrap();
}
#[test]
fn sync_all_report_output_write_jsonl_emits_exactly_one_line() {
let output = SyncAllReportOutput {
accounts: vec![AccountSyncOutcome {
account: "acct-a".to_string(),
account_error: None,
actions: Vec::new(),
errors: Vec::new(),
summary: SyncSummary::default(),
}],
combined_summary: SyncSummary::default(),
};
let mut buf = Vec::new();
output.write_jsonl(&mut buf).unwrap();
let text = String::from_utf8(buf).unwrap();
assert_eq!(text.lines().count(), 1);
assert!(text.contains("acct-a"));
}
}