pigeon/commands/job/cli.rs
1use std::path::PathBuf;
2
3use clap::{Args, Subcommand};
4
5use crate::core::observability::Observable;
6
7#[derive(Args, Debug)]
8pub struct JobArgs {
9 #[command(subcommand)]
10 pub command: JobCommands,
11}
12
13#[derive(Subcommand, Debug)]
14pub enum JobCommands {
15 /// Run a job (see subcommands for available job types)
16 Run(RunArgs),
17}
18
19#[derive(Args, Debug)]
20pub struct RunArgs {
21 #[command(subcommand)]
22 pub job_type: JobType,
23}
24
25#[derive(Subcommand, Debug)]
26pub enum JobType {
27 /// Fetch, transform, deduplicate, and optionally upload mail for one or
28 /// more authenticated email identities. Replaces `pigeon email sync`
29 /// (ADR-0021).
30 EmailSync {
31 /// Aliases of the identities to sync, comma-separated. Interactively
32 /// selected from the authenticated identities when omitted and
33 /// stdin is a terminal; required otherwise.
34 #[arg(long, value_delimiter = ',')]
35 identities: Option<Vec<String>>,
36
37 /// Local directory to stage and store output under, shared across
38 /// every selected identity (each gets its own subdirectory
39 /// underneath). Defaults to a directory under the OS temp directory
40 /// when omitted.
41 #[arg(long)]
42 local_output: Option<PathBuf>,
43
44 /// Alias of a configured bucket-config (see `pigeon dataops
45 /// bucket-config new`) to upload each identity's local result tree
46 /// to, once its local fetch/transform/dedupe phase is complete.
47 #[arg(long)]
48 remote_output: Option<String>,
49
50 /// Alias of a configured encryption key (see `pigeon keyring add
51 /// encryption-key`) to use, overriding the target bucket-config's
52 /// own default (if any). Interactively selected/confirmed when
53 /// omitted; falls back to the bucket's default non-interactively
54 /// (ADR-0027).
55 #[arg(long)]
56 encryption_key: Option<String>,
57
58 /// Maximum number of fetch/transform workers to run concurrently,
59 /// spanning every selected identity's every mailbox. Interactively
60 /// prompted (with a rough time estimate) when omitted and stdin is
61 /// a terminal; required otherwise.
62 #[arg(long)]
63 concurrency: Option<usize>,
64
65 /// Maximum number of files to upload concurrently, independent of
66 /// `--concurrency` (which sizes IMAP fetch/transform work) -- the
67 /// upload phase is network-round-trip-bound, not IMAP-bound, so it
68 /// benefits from its own, separately-tuned concurrency (ADR-0091).
69 /// Interactively prompted when omitted and stdin is a terminal;
70 /// defaults to 16 otherwise. With `--upload-only`, this is the only
71 /// concurrency flag that has any effect.
72 #[arg(long)]
73 upload_concurrency: Option<usize>,
74
75 /// Maximum number of simultaneous IMAP connections opened to any
76 /// one identity, regardless of `--concurrency` (ADR-0071) -- caps
77 /// worker concurrency per-account rather than only globally, so a
78 /// mailbox with enough pending batches can't cause more than this
79 /// many workers to log in to the same account at once and trip a
80 /// provider's simultaneous-connection limit. Defaults to 6 (well
81 /// under Gmail's documented 15-connection cap) when omitted; not
82 /// interactively prompted.
83 #[arg(long)]
84 max_connections_per_identity: Option<usize>,
85
86 /// Resumes uploading already-completed local runs for the selected
87 /// identities instead of starting new ones: skips the IMAP
88 /// connect/fetch/transform/dedup phases entirely (and the
89 /// per-identity IMAP credentials they'd otherwise need) and
90 /// uploads straight from each identity's existing local result
91 /// tree, picking up where a prior run's upload phase left off via
92 /// the same `.staging/.uploaded` index per identity (ADR-0090). An
93 /// identity with no completed local run is skipped with a warning
94 /// rather than failing the whole command. Requires a mandatory
95 /// `--remote-output`.
96 #[arg(long)]
97 upload_only: bool,
98
99 /// Alias of a configured bucket-config this run's report, the
100 /// shared observability log, and a transcript of its printed
101 /// output are uploaded to, always unencrypted, under a
102 /// `YYYY-MM-DD-job-name-{run-id}/` prefix (ADR-0100). Mandatory --
103 /// interactively selected when omitted and stdin is a terminal;
104 /// required otherwise.
105 #[arg(long)]
106 report_bucket: Option<String>,
107
108 /// Suppress every wizard prompt (ADR-0114): each omitted input
109 /// resolves via its non-interactive fallback (or errors if none
110 /// exists) instead of asking, and the final "proceed?"
111 /// confirmation is auto-answered yes.
112 #[arg(long)]
113 non_interactive: bool,
114 },
115
116 /// Fetches raw `.eml` files and unpacked attachments (no Markdown/
117 /// frontmatter transform) for one or more authenticated email
118 /// identities, deduplicating attachments by content, and optionally
119 /// uploads the result unencrypted to a bucket-config (ADR-0081).
120 EmailPull {
121 /// Aliases of the identities to pull, comma-separated.
122 /// Interactively selected from the authenticated identities when
123 /// omitted and stdin is a terminal; required otherwise.
124 #[arg(long, value_delimiter = ',')]
125 identities: Option<Vec<String>>,
126
127 /// Local directory to stage and store output under, shared across
128 /// every selected identity. Defaults to a directory under the OS
129 /// temp directory when omitted.
130 #[arg(long)]
131 local_output: Option<PathBuf>,
132
133 /// Alias of a configured bucket-config to upload each identity's
134 /// local result tree to, once its local fetch/dedupe phase is
135 /// complete. Always uploaded unencrypted -- this job never offers
136 /// encryption (ADR-0081).
137 #[arg(long)]
138 remote_output: Option<String>,
139
140 /// Maximum number of fetch/extract workers to run concurrently,
141 /// spanning every selected identity's every mailbox. Interactively
142 /// prompted (with a rough time estimate) when omitted and stdin is
143 /// a terminal; required otherwise.
144 #[arg(long)]
145 concurrency: Option<usize>,
146
147 /// Maximum number of files to upload concurrently, independent of
148 /// `--concurrency` (which sizes IMAP fetch/extract work) -- the
149 /// upload phase is network-round-trip-bound, not IMAP-bound, so it
150 /// benefits from its own, separately-tuned concurrency (ADR-0091).
151 /// Interactively prompted when omitted and stdin is a terminal;
152 /// defaults to 16 otherwise. With `--upload-only`, this is the only
153 /// concurrency flag that has any effect.
154 #[arg(long)]
155 upload_concurrency: Option<usize>,
156
157 /// Maximum number of simultaneous IMAP connections opened to any
158 /// one identity, regardless of `--concurrency`. Defaults to 6 when
159 /// omitted; not interactively prompted.
160 #[arg(long)]
161 max_connections_per_identity: Option<usize>,
162
163 /// Resumes uploading already-completed local runs for the selected
164 /// identities instead of starting new ones: skips the IMAP
165 /// connect/fetch/dedup phases entirely (and the per-identity IMAP
166 /// credentials they'd otherwise need) and uploads straight from
167 /// each identity's existing local result tree, picking up where a
168 /// prior run's upload phase left off via the same
169 /// `.staging/.uploaded` index per identity (ADR-0090). An identity
170 /// with no completed local run is skipped with a warning rather
171 /// than failing the whole command. Requires a mandatory
172 /// `--remote-output`.
173 #[arg(long)]
174 upload_only: bool,
175
176 /// Alias of a configured bucket-config this run's report, the
177 /// shared observability log, and a transcript of its printed
178 /// output are uploaded to, always unencrypted, under a
179 /// `YYYY-MM-DD-job-name-{run-id}/` prefix (ADR-0100). Mandatory --
180 /// interactively selected when omitted and stdin is a terminal;
181 /// required otherwise.
182 #[arg(long)]
183 report_bucket: Option<String>,
184
185 /// Suppress every wizard prompt (ADR-0114): each omitted input
186 /// resolves via its non-interactive fallback (or errors if none
187 /// exists) instead of asking, and the final "proceed?"
188 /// confirmation is auto-answered yes.
189 #[arg(long)]
190 non_interactive: bool,
191 },
192
193 /// Recursively scans a bucket, always inflates every zip found (the
194 /// zip container itself is never uploaded, only its inflated
195 /// contents), content-hashes every file bucket-wide to keep one
196 /// byte-identical copy of each, writes a human-readable merge report,
197 /// and optionally uploads the result unencrypted to a (possibly
198 /// different) bucket-config (ADR-0082). Every file is always processed
199 /// and every zip is always expanded -- there is no file-type or
200 /// zip-expansion selection, and this job never offers encryption.
201 Deduplicate {
202 /// Alias of a configured bucket-config to pull from. Repeat the
203 /// flag for more than one (`--source-bucket a --source-bucket b`)
204 /// -- every listed bucket is downloaded into one shared local
205 /// staging tree and deduplicated across the combined set, not
206 /// per-bucket (ADR-0109). Interactively multi-selected from the
207 /// configured bucket-configs when omitted and stdin is a terminal;
208 /// at least one is required otherwise.
209 #[arg(long)]
210 source_bucket: Vec<String>,
211
212 /// Local directory to stage and store output under. Defaults to a
213 /// directory under the OS temp directory when omitted.
214 #[arg(long)]
215 local_output: Option<PathBuf>,
216
217 /// Alias of a configured bucket-config to upload the deduped
218 /// result to, once local processing is complete. Always uploaded
219 /// unencrypted. Single bucket only (ADR-0109).
220 #[arg(long)]
221 destination_bucket: Option<String>,
222
223 /// Maximum number of files to download/hash concurrently.
224 #[arg(long)]
225 concurrency: Option<usize>,
226
227 /// rclone --transfers for the upload phase, now one bulk `rclone
228 /// copy` rather than a per-file task queue (ADR-0136). Defaults to
229 /// 8 when omitted. With `--upload-only`, this is the only transfer
230 /// concurrency flag that has any effect.
231 #[arg(long)]
232 transfers: Option<usize>,
233
234 /// rclone --checkers for the upload phase. Defaults to 16 when
235 /// omitted.
236 #[arg(long)]
237 checkers: Option<usize>,
238
239 /// rclone --tpslimit for the upload phase. Unset by default -- no
240 /// rate cap.
241 #[arg(long)]
242 tpslimit: Option<usize>,
243
244 /// Resumes uploading an already-completed local deduplicate run instead
245 /// of starting a new one: skips the bucket listing/download/hash/
246 /// placement phases entirely (and the source bucket credentials
247 /// they'd otherwise need) and uploads straight from an existing
248 /// `--local-output`'s `result/` tree, picking up where a prior
249 /// run's upload phase left off via the same `.staging/.uploaded`
250 /// index (ADR-0089). Requires a `--local-output` from a completed
251 /// prior run (its `.staging/.processed` checkpoint must exist and
252 /// its `result/` must be non-empty) and a mandatory
253 /// `--remote-output`.
254 #[arg(long)]
255 upload_only: bool,
256
257 /// Alias of a configured bucket-config this run's report, the
258 /// shared observability log, and a transcript of its printed
259 /// output are uploaded to, always unencrypted, under a
260 /// `YYYY-MM-DD-job-name-{run-id}/` prefix (ADR-0100). Mandatory --
261 /// interactively selected when omitted and stdin is a terminal;
262 /// required otherwise.
263 #[arg(long)]
264 report_bucket: Option<String>,
265
266 /// Suppress every wizard prompt (ADR-0114): each omitted input
267 /// resolves via its non-interactive fallback (or errors if none
268 /// exists) instead of asking, and the final "proceed?"
269 /// confirmation is auto-answered yes.
270 #[arg(long)]
271 non_interactive: bool,
272 },
273
274 /// Shells out to the external `rclone` binary for copy/delete
275 /// operations against pigeon keyring `BucketConfig` aliases. `rclone`
276 /// is always invoked against a config file pigeon itself generates
277 /// from the keyring (ADR-0136, reversing ADR-0101/ADR-0110's original
278 /// raw-passthrough design) -- never an externally-provisioned or
279 /// user-general-purpose `rclone.conf`. See subcommands for the
280 /// available actions. Requires `rclone` on `PATH`.
281 Rclone(RcloneArgs),
282
283 /// Transcodes image files (`png`/`jpeg`/`heic`) into full-size,
284 /// maximum-quality `.jpg`, or video files (`mov`/`m4v`/`mp4`, ADR-0122)
285 /// into space-optimized H.265/HEVC `.mp4` at a chosen `--video-quality`
286 /// (ADR-0112, pipeline reworked by ADR-0116): shells out to `rclone` to
287 /// pull `--source` into a local staging tree, transcoding/copying each
288 /// file locally and pushing it individually via `rclone copyto` the
289 /// moment it's ready, rather than waiting for the whole batch.
290 /// `--source`/`--destination` are pigeon keyring `BucketConfig`
291 /// aliases, optionally composed with a prefix within that bucket
292 /// (`alias:prefix/`, ADR-0138); an omitted prefix defaults to
293 /// `--input-file-type`'s own extension (source) or output extension
294 /// (destination). Deduplication is never involved -- every output file's
295 /// name is derived deterministically and is unique by construction;
296 /// duplicate source content is deliberately retained at separate
297 /// destination names. A single file's transcode or push failure (e.g.
298 /// a HEIC file the local `ffmpeg` build can't decode, retried before
299 /// giving up) is recorded and reported but never stops any other file
300 /// from being processed and pushed; every file that does succeed stays
301 /// checkpointed for a resumed rerun. Requires both `rclone` and
302 /// `ffmpeg`/`ffprobe` on `PATH`.
303 Transform {
304 /// Which input format this run transcodes: `png`, `jpeg`, `heic`,
305 /// `mov`, `m4v`, or `mp4`. Interactively selected when omitted and
306 /// stdin is a terminal; required otherwise.
307 #[arg(long)]
308 input_file_type: Option<String>,
309
310 /// Compression level for a video input kind (`mov`/`m4v`/`mp4`):
311 /// `low`, `medium` (default), `high`, or `lossless` -- a small
312 /// vetted `libx265` CRF menu (ADR-0122). Ignored for an image input
313 /// kind. Interactively selected when omitted, stdin is a terminal,
314 /// and the input kind is video; defaults to `medium` otherwise.
315 #[arg(long)]
316 video_quality: Option<String>,
317
318 /// Where to pull from: a configured bucket-config alias, optionally
319 /// composed with a prefix inside it -- `media` is the whole bucket,
320 /// `media:mp4/` its `mp4/` directory (ADR-0138). An omitted prefix
321 /// defaults to `--input-file-type`'s own extension, so
322 /// `--source media --input-file-type mp4` means `media:mp4/`; an
323 /// explicit prefix is always used verbatim. Interactively selected
324 /// (bucket root, then the default prefix) when omitted and stdin is
325 /// a terminal; required otherwise.
326 #[arg(long)]
327 source: Option<String>,
328
329 /// Where to push to, same `alias[:prefix/]` form as `--source`
330 /// (ADR-0138). An omitted prefix defaults to `--input-file-type`'s
331 /// *output* extension (`jpg` for an image kind, `mp4` for a video
332 /// one), so one bucket can serve as both source and destination
333 /// when the prefixes differ.
334 #[arg(long)]
335 destination: Option<String>,
336
337 /// Local directory this run stages pulled source files, transcoded
338 /// results, and bookkeeping under. Defaults to a directory under
339 /// the OS temp directory when omitted.
340 #[arg(long)]
341 local_output: Option<PathBuf>,
342
343 /// Maximum number of files to transcode concurrently (the ffmpeg
344 /// phase only -- `rclone` manages its own internal parallelism via
345 /// `--download-transfers`/`--download-checkers`). Interactively
346 /// prompted (a cores-based default) when omitted and stdin is a
347 /// terminal; required otherwise.
348 #[arg(long)]
349 concurrency: Option<usize>,
350
351 /// rclone --transfers for the download (pull) phase. Defaults to 8
352 /// when omitted.
353 #[arg(long)]
354 download_transfers: Option<usize>,
355
356 /// rclone --checkers for the download (pull) phase. Defaults to 16
357 /// when omitted.
358 #[arg(long)]
359 download_checkers: Option<usize>,
360
361 /// Maximum number of concurrent per-file push (`rclone copyto`)
362 /// operations. Defaults to 8 when omitted. Separate from
363 /// `--download-transfers` to allow independent tuning of pull vs.
364 /// push phases (ADR-0129).
365 #[arg(long)]
366 upload_transfers: Option<usize>,
367
368 /// Maximum number of downloaded but not-yet-processed files
369 /// (pending transcode/push) before the pull phase is paused to allow
370 /// the pipeline to catch up. Defaults to `concurrency * 4` when
371 /// omitted. Enables automatic disk-pressure backpressure without
372 /// requiring manual tuning (ADR-0129).
373 #[arg(long)]
374 max_pending_downloads: Option<usize>,
375
376 /// rclone --tpslimit, used by the pull phase only. Unset by
377 /// default -- no rate cap.
378 #[arg(long)]
379 tpslimit: Option<usize>,
380
381 /// Alias of a configured bucket-config this run's report, the
382 /// shared observability log, and a transcript of its printed
383 /// output are uploaded to, always unencrypted, under a
384 /// `YYYY-MM-DD-job-name-{run-id}/` prefix (ADR-0100). Mandatory --
385 /// interactively selected when omitted and stdin is a terminal;
386 /// required otherwise.
387 #[arg(long)]
388 report_bucket: Option<String>,
389
390 /// Suppress every wizard prompt (ADR-0114): each omitted input
391 /// resolves via its non-interactive fallback (or errors if none
392 /// exists) instead of asking, and the final "proceed?"
393 /// confirmation is auto-answered yes.
394 #[arg(long)]
395 non_interactive: bool,
396 },
397}
398
399#[derive(Args, Debug)]
400pub struct RcloneArgs {
401 #[command(subcommand)]
402 pub action: RcloneAction,
403}
404
405#[derive(Subcommand, Debug)]
406pub enum RcloneAction {
407 /// Copies data from one configured bucket-config to another by
408 /// shelling out to `rclone copy`, against a config file pigeon itself
409 /// generates from the keyring (ADR-0136, reversing ADR-0101's original
410 /// raw-passthrough design). `--transfers`/`--checkers`/`--tpslimit` are
411 /// overridable per-destination (ADR-0108); every other rclone
412 /// performance/retry flag stays fixed.
413 Copy {
414 /// Where to pull from: a configured bucket-config alias, optionally
415 /// composed with a prefix inside it -- `media` is the whole bucket,
416 /// `media:mp4/` its `mp4/` directory (ADR-0138). Unlike
417 /// `transform`, an omitted prefix stays the bucket root (this job
418 /// has no file-type concept to derive one from). Interactively
419 /// selected when omitted and stdin is a terminal; required
420 /// otherwise.
421 #[arg(long)]
422 source_bucket: Option<String>,
423
424 /// Where to push to, same `alias[:prefix/]` form as
425 /// `--source-bucket` (ADR-0138). Interactively selected when
426 /// omitted and stdin is a terminal; required otherwise.
427 #[arg(long)]
428 destination_bucket: Option<String>,
429
430 /// Local directory this run's rclone log (also serving as this
431 /// job's report) and transcript are written under. Defaults to a
432 /// directory under the OS temp directory when omitted. Unlike
433 /// every other job, this is not a staging area for transferred
434 /// data -- rclone transfers directly source -> destination with no
435 /// pigeon-side staging.
436 #[arg(long)]
437 local_output: Option<PathBuf>,
438
439 /// Alias of a configured bucket-config this run's report (the
440 /// rclone log itself, for this job), the shared observability log,
441 /// and a transcript of its printed output are uploaded to, always
442 /// unencrypted, under a `YYYY-MM-DD-job-name-{run-id}/` prefix
443 /// (ADR-0100). Mandatory -- interactively selected when omitted
444 /// and stdin is a terminal; required otherwise.
445 #[arg(long)]
446 report_bucket: Option<String>,
447
448 /// rclone --transfers (concurrent file transfers). Defaults to 8
449 /// (ADR-0106) when omitted.
450 #[arg(long)]
451 transfers: Option<usize>,
452
453 /// rclone --checkers (concurrent list/compare operations).
454 /// Defaults to 16 (ADR-0106) when omitted.
455 #[arg(long)]
456 checkers: Option<usize>,
457
458 /// rclone --tpslimit (transactions/sec ceiling across transfers
459 /// and checkers combined). Unset by default -- no rate cap --
460 /// unless a specific destination needs one (ADR-0108).
461 #[arg(long)]
462 tpslimit: Option<usize>,
463
464 /// Suppress every wizard prompt (ADR-0114): each omitted input
465 /// resolves via its non-interactive fallback (or errors if none
466 /// exists) instead of asking, and the final "proceed?"
467 /// confirmation is auto-answered yes.
468 #[arg(long)]
469 non_interactive: bool,
470 },
471
472 /// Recursively and permanently deletes everything in a configured
473 /// bucket-config by shelling out to `rclone purge` (ADR-0110).
474 /// Irreversible -- there is no `--dry-run` and no undo beyond the
475 /// existing `--non-interactive`/interactive confirmation. Unlike
476 /// `copy`, there is no destination and no per-file transfer-rate
477 /// tuning, since purge transfers no file content.
478 Delete {
479 /// What to recursively and permanently delete the contents of: a
480 /// configured bucket-config alias, optionally composed with a
481 /// prefix to scope the purge to one directory inside it
482 /// (`media:mp4/`, ADR-0138). An omitted prefix purges the **whole
483 /// bucket**. Interactively selected when omitted and stdin is a
484 /// terminal; required otherwise.
485 #[arg(long)]
486 source_bucket: Option<String>,
487
488 /// Local directory this run's rclone log (also serving as this
489 /// job's report) and transcript are written under. Defaults to a
490 /// directory under the OS temp directory when omitted.
491 #[arg(long)]
492 local_output: Option<PathBuf>,
493
494 /// Alias of a configured bucket-config this run's report, the
495 /// shared observability log, and a transcript of its printed
496 /// output are uploaded to, always unencrypted, under a
497 /// `YYYY-MM-DD-job-name-{run-id}/` prefix (ADR-0100). Mandatory --
498 /// interactively selected when omitted and stdin is a terminal;
499 /// required otherwise.
500 #[arg(long)]
501 report_bucket: Option<String>,
502
503 /// rclone --checkers (concurrent list operations during
504 /// enumeration). Defaults to 16 when omitted.
505 #[arg(long)]
506 checkers: Option<usize>,
507
508 /// Suppress every wizard prompt (ADR-0114): each omitted input
509 /// resolves via its non-interactive fallback (or errors if none
510 /// exists) instead of asking, and the final "proceed?"
511 /// confirmation is auto-answered yes.
512 #[arg(long)]
513 non_interactive: bool,
514 },
515}
516
517impl Observable for JobType {
518 fn command_name(&self) -> &'static str {
519 match self {
520 JobType::EmailSync { .. } => "job.email-sync",
521 JobType::EmailPull { .. } => "job.email-pull",
522 JobType::Deduplicate { .. } => "job.deduplicate",
523 JobType::Rclone(args) => match args.action {
524 RcloneAction::Copy { .. } => "job.rclone-copy",
525 RcloneAction::Delete { .. } => "job.rclone-delete",
526 },
527 JobType::Transform { .. } => "job.transform",
528 }
529 }
530}
531
532impl JobType {
533 /// The bare dash-form job name (e.g. `"deduplicate"`) used in the
534 /// report-bucket upload prefix (ADR-0100) -- `command_name()`'s value
535 /// with its `"job."` prefix stripped, rather than a 6th place in this
536 /// codebase repeating the same job-name literals already duplicated
537 /// across every job's `worker.rs` metrics call sites.
538 pub(crate) fn job_name(&self) -> &'static str {
539 self.command_name().trim_start_matches("job.")
540 }
541}