pigeon/commands/job/cli.rs
1use std::path::PathBuf;
2
3use clap::{Args, Subcommand};
4
5use crate::core::observability::Observable;
6
7#[derive(Args, Debug)]
8pub struct JobArgs {
9 #[command(subcommand)]
10 pub command: JobCommands,
11}
12
13#[derive(Subcommand, Debug)]
14pub enum JobCommands {
15 /// Run a job (see subcommands for available job types)
16 Run(RunArgs),
17}
18
19#[derive(Args, Debug)]
20pub struct RunArgs {
21 #[command(subcommand)]
22 pub job_type: JobType,
23}
24
25#[derive(Subcommand, Debug)]
26pub enum JobType {
27 /// Fetch, transform, deduplicate, and optionally upload mail for one or
28 /// more authenticated email identities. Replaces `pigeon email sync`
29 /// (ADR-0021).
30 EmailSync {
31 /// Aliases of the identities to sync, comma-separated. Interactively
32 /// selected from the authenticated identities when omitted and
33 /// stdin is a terminal; required otherwise.
34 #[arg(long, value_delimiter = ',')]
35 identities: Option<Vec<String>>,
36
37 /// Local directory to stage and store output under, shared across
38 /// every selected identity (each gets its own subdirectory
39 /// underneath). Defaults to a directory under the OS temp directory
40 /// when omitted.
41 #[arg(long)]
42 local_output: Option<PathBuf>,
43
44 /// Alias of a configured bucket-config (see `pigeon dataops
45 /// bucket-config new`) to upload each identity's local result tree
46 /// to, once its local fetch/transform/dedupe phase is complete.
47 #[arg(long)]
48 remote_output: Option<String>,
49
50 /// Alias of a configured encryption key (see `pigeon keyring add
51 /// encryption-key`) to use, overriding the target bucket-config's
52 /// own default (if any). Interactively selected/confirmed when
53 /// omitted; falls back to the bucket's default non-interactively
54 /// (ADR-0027).
55 #[arg(long)]
56 encryption_key: Option<String>,
57
58 /// Maximum number of fetch/transform workers to run concurrently,
59 /// spanning every selected identity's every mailbox. Interactively
60 /// prompted (with a rough time estimate) when omitted and stdin is
61 /// a terminal; required otherwise.
62 #[arg(long)]
63 concurrency: Option<usize>,
64
65 /// Maximum number of files to upload concurrently, independent of
66 /// `--concurrency` (which sizes IMAP fetch/transform work) -- the
67 /// upload phase is network-round-trip-bound, not IMAP-bound, so it
68 /// benefits from its own, separately-tuned concurrency (ADR-0091).
69 /// Interactively prompted when omitted and stdin is a terminal;
70 /// defaults to 16 otherwise. With `--upload-only`, this is the only
71 /// concurrency flag that has any effect.
72 #[arg(long)]
73 upload_concurrency: Option<usize>,
74
75 /// Maximum number of simultaneous IMAP connections opened to any
76 /// one identity, regardless of `--concurrency` (ADR-0071) -- caps
77 /// worker concurrency per-account rather than only globally, so a
78 /// mailbox with enough pending batches can't cause more than this
79 /// many workers to log in to the same account at once and trip a
80 /// provider's simultaneous-connection limit. Defaults to 6 (well
81 /// under Gmail's documented 15-connection cap) when omitted; not
82 /// interactively prompted.
83 #[arg(long)]
84 max_connections_per_identity: Option<usize>,
85
86 /// Resumes uploading already-completed local runs for the selected
87 /// identities instead of starting new ones: skips the IMAP
88 /// connect/fetch/transform/dedup phases entirely (and the
89 /// per-identity IMAP credentials they'd otherwise need) and
90 /// uploads straight from each identity's existing local result
91 /// tree, picking up where a prior run's upload phase left off via
92 /// the same `.staging/.uploaded` index per identity (ADR-0090). An
93 /// identity with no completed local run is skipped with a warning
94 /// rather than failing the whole command. Requires a mandatory
95 /// `--remote-output`.
96 #[arg(long)]
97 upload_only: bool,
98
99 /// Alias of a configured bucket-config this run's report, the
100 /// shared observability log, and a transcript of its printed
101 /// output are uploaded to, always unencrypted, under a
102 /// `YYYY-MM-DD-job-name-{run-id}/` prefix (ADR-0100). Mandatory --
103 /// interactively selected when omitted and stdin is a terminal;
104 /// required otherwise.
105 #[arg(long)]
106 report_bucket: Option<String>,
107
108 /// Suppress every wizard prompt (ADR-0114): each omitted input
109 /// resolves via its non-interactive fallback (or errors if none
110 /// exists) instead of asking, and the final "proceed?"
111 /// confirmation is auto-answered yes.
112 #[arg(long)]
113 non_interactive: bool,
114 },
115
116 /// Decrypts every `*.enc` file under `--input-dir` into `--output-dir`
117 /// (`.enc` suffix stripped, relative structure preserved), using a
118 /// configured encryption key (ADR-0028).
119 DecryptFiles {
120 /// Directory containing `*.enc` files to decrypt. Interactively
121 /// prompted when omitted and stdin is a terminal; required
122 /// otherwise.
123 #[arg(long)]
124 input_dir: Option<PathBuf>,
125
126 /// Directory decrypted files are written under, mirroring
127 /// `--input-dir`'s relative structure. Must not be the same
128 /// directory as `--input-dir`. Interactively prompted when omitted
129 /// and stdin is a terminal; required otherwise.
130 #[arg(long)]
131 output_dir: Option<PathBuf>,
132
133 /// Alias of a configured encryption key (see `pigeon keyring add
134 /// encryption-key`) to decrypt with. Interactively selected when
135 /// omitted and stdin is a terminal; required otherwise.
136 #[arg(long)]
137 encryption_key: Option<String>,
138
139 /// Maximum number of files to decrypt concurrently.
140 #[arg(long)]
141 concurrency: Option<usize>,
142
143 /// Alias of a configured bucket-config this run's report, the
144 /// shared observability log, and a transcript of its printed
145 /// output are uploaded to, always unencrypted, under a
146 /// `YYYY-MM-DD-job-name-{run-id}/` prefix (ADR-0100). Mandatory --
147 /// interactively selected when omitted and stdin is a terminal;
148 /// required otherwise.
149 #[arg(long)]
150 report_bucket: Option<String>,
151
152 /// Suppress every wizard prompt (ADR-0114): each omitted input
153 /// resolves via its non-interactive fallback (or errors if none
154 /// exists) instead of asking, and the final "proceed?"
155 /// confirmation is auto-answered yes.
156 #[arg(long)]
157 non_interactive: bool,
158 },
159
160 /// Fetches raw `.eml` files and unpacked attachments (no Markdown/
161 /// frontmatter transform) for one or more authenticated email
162 /// identities, deduplicating attachments by content, and optionally
163 /// uploads the result unencrypted to a bucket-config (ADR-0081).
164 EmailPull {
165 /// Aliases of the identities to pull, comma-separated.
166 /// Interactively selected from the authenticated identities when
167 /// omitted and stdin is a terminal; required otherwise.
168 #[arg(long, value_delimiter = ',')]
169 identities: Option<Vec<String>>,
170
171 /// Local directory to stage and store output under, shared across
172 /// every selected identity. Defaults to a directory under the OS
173 /// temp directory when omitted.
174 #[arg(long)]
175 local_output: Option<PathBuf>,
176
177 /// Alias of a configured bucket-config to upload each identity's
178 /// local result tree to, once its local fetch/dedupe phase is
179 /// complete. Always uploaded unencrypted -- this job never offers
180 /// encryption (ADR-0081).
181 #[arg(long)]
182 remote_output: Option<String>,
183
184 /// Maximum number of fetch/extract workers to run concurrently,
185 /// spanning every selected identity's every mailbox. Interactively
186 /// prompted (with a rough time estimate) when omitted and stdin is
187 /// a terminal; required otherwise.
188 #[arg(long)]
189 concurrency: Option<usize>,
190
191 /// Maximum number of files to upload concurrently, independent of
192 /// `--concurrency` (which sizes IMAP fetch/extract work) -- the
193 /// upload phase is network-round-trip-bound, not IMAP-bound, so it
194 /// benefits from its own, separately-tuned concurrency (ADR-0091).
195 /// Interactively prompted when omitted and stdin is a terminal;
196 /// defaults to 16 otherwise. With `--upload-only`, this is the only
197 /// concurrency flag that has any effect.
198 #[arg(long)]
199 upload_concurrency: Option<usize>,
200
201 /// Maximum number of simultaneous IMAP connections opened to any
202 /// one identity, regardless of `--concurrency`. Defaults to 6 when
203 /// omitted; not interactively prompted.
204 #[arg(long)]
205 max_connections_per_identity: Option<usize>,
206
207 /// Resumes uploading already-completed local runs for the selected
208 /// identities instead of starting new ones: skips the IMAP
209 /// connect/fetch/dedup phases entirely (and the per-identity IMAP
210 /// credentials they'd otherwise need) and uploads straight from
211 /// each identity's existing local result tree, picking up where a
212 /// prior run's upload phase left off via the same
213 /// `.staging/.uploaded` index per identity (ADR-0090). An identity
214 /// with no completed local run is skipped with a warning rather
215 /// than failing the whole command. Requires a mandatory
216 /// `--remote-output`.
217 #[arg(long)]
218 upload_only: bool,
219
220 /// Alias of a configured bucket-config this run's report, the
221 /// shared observability log, and a transcript of its printed
222 /// output are uploaded to, always unencrypted, under a
223 /// `YYYY-MM-DD-job-name-{run-id}/` prefix (ADR-0100). Mandatory --
224 /// interactively selected when omitted and stdin is a terminal;
225 /// required otherwise.
226 #[arg(long)]
227 report_bucket: Option<String>,
228
229 /// Suppress every wizard prompt (ADR-0114): each omitted input
230 /// resolves via its non-interactive fallback (or errors if none
231 /// exists) instead of asking, and the final "proceed?"
232 /// confirmation is auto-answered yes.
233 #[arg(long)]
234 non_interactive: bool,
235 },
236
237 /// Recursively scans a bucket, always inflates every zip found (the
238 /// zip container itself is never uploaded, only its inflated
239 /// contents), content-hashes every file bucket-wide to keep one
240 /// byte-identical copy of each, writes a human-readable merge report,
241 /// and optionally uploads the result unencrypted to a (possibly
242 /// different) bucket-config (ADR-0082). Every file is always processed
243 /// and every zip is always expanded -- there is no file-type or
244 /// zip-expansion selection, and this job never offers encryption.
245 Deduplicate {
246 /// Alias of a configured bucket-config to pull from. Repeat the
247 /// flag for more than one (`--source-bucket a --source-bucket b`)
248 /// -- every listed bucket is downloaded into one shared local
249 /// staging tree and deduplicated across the combined set, not
250 /// per-bucket (ADR-0109). Interactively multi-selected from the
251 /// configured bucket-configs when omitted and stdin is a terminal;
252 /// at least one is required otherwise.
253 #[arg(long)]
254 source_bucket: Vec<String>,
255
256 /// Local directory to stage and store output under. Defaults to a
257 /// directory under the OS temp directory when omitted.
258 #[arg(long)]
259 local_output: Option<PathBuf>,
260
261 /// Alias of a configured bucket-config to upload the deduped
262 /// result to, once local processing is complete. Always uploaded
263 /// unencrypted. Single bucket only (ADR-0109).
264 #[arg(long)]
265 destination_bucket: Option<String>,
266
267 /// Maximum number of files to download/hash concurrently.
268 #[arg(long)]
269 concurrency: Option<usize>,
270
271 /// Maximum number of files to upload concurrently, independent of
272 /// `--concurrency` (which sizes download/hash work) -- the upload
273 /// phase is network-round-trip-bound, not CPU-bound, so it benefits
274 /// from its own, separately-tuned concurrency (ADR-0091).
275 /// Interactively prompted when omitted and stdin is a terminal;
276 /// defaults to 16 otherwise. With `--upload-only`, this is the only
277 /// concurrency flag that has any effect.
278 #[arg(long)]
279 upload_concurrency: Option<usize>,
280
281 /// Resumes uploading an already-completed local deduplicate run instead
282 /// of starting a new one: skips the bucket listing/download/hash/
283 /// placement phases entirely (and the source bucket credentials
284 /// they'd otherwise need) and uploads straight from an existing
285 /// `--local-output`'s `result/` tree, picking up where a prior
286 /// run's upload phase left off via the same `.staging/.uploaded`
287 /// index (ADR-0089). Requires a `--local-output` from a completed
288 /// prior run (its `.staging/.processed` checkpoint must exist and
289 /// its `result/` must be non-empty) and a mandatory
290 /// `--remote-output`.
291 #[arg(long)]
292 upload_only: bool,
293
294 /// Alias of a configured bucket-config this run's report, the
295 /// shared observability log, and a transcript of its printed
296 /// output are uploaded to, always unencrypted, under a
297 /// `YYYY-MM-DD-job-name-{run-id}/` prefix (ADR-0100). Mandatory --
298 /// interactively selected when omitted and stdin is a terminal;
299 /// required otherwise.
300 #[arg(long)]
301 report_bucket: Option<String>,
302
303 /// Suppress every wizard prompt (ADR-0114): each omitted input
304 /// resolves via its non-interactive fallback (or errors if none
305 /// exists) instead of asking, and the final "proceed?"
306 /// confirmation is auto-answered yes.
307 #[arg(long)]
308 non_interactive: bool,
309 },
310
311 /// Shells out to the external `rclone` binary for copy/delete
312 /// operations against raw rclone `remote:path` strings. Pigeon manages
313 /// no rclone credentials/config -- `rclone.conf` is provisioned by an
314 /// external process, outside this crate's scope (ADR-0101, restructured
315 /// by ADR-0110). See subcommands for the available actions. Requires
316 /// `rclone` on `PATH`.
317 Rclone(RcloneArgs),
318
319 /// Transcodes image files (`png`/`jpeg`/`heic`) into full-size,
320 /// maximum-quality `.jpg`, or video files (`mov`/`m4v`/`mp4`, ADR-0122)
321 /// into space-optimized H.265/HEVC `.mp4` at a chosen `--video-quality`
322 /// (ADR-0112, pipeline reworked by ADR-0116): shells out to `rclone` to
323 /// pull `--source-path` into a local staging tree, transcoding/copying
324 /// each file locally and pushing it individually via `rclone copyto`
325 /// the moment it's ready, rather than waiting for the whole batch.
326 /// `--source-path`/`--destination-path` are raw rclone `remote:path`
327 /// strings, same semantics as the `rclone` job's own flags of the same
328 /// name -- never pigeon `BucketConfig`/keyring aliases. Deduplication
329 /// is never involved -- every output file's name is derived
330 /// deterministically and is unique by construction; duplicate source
331 /// content is deliberately retained at separate destination names. A
332 /// single file's transcode or push failure (e.g. a HEIC file the local
333 /// `ffmpeg` build can't decode, retried before giving up) is recorded
334 /// and reported but never stops any other file from being processed
335 /// and pushed; every file that does succeed stays checkpointed for a
336 /// resumed rerun. Requires both `rclone` and `ffmpeg`/`ffprobe` on
337 /// `PATH`.
338 Transform {
339 /// Which input format this run transcodes: `png`, `jpeg`, `heic`,
340 /// `mov`, `m4v`, or `mp4`. Interactively selected when omitted and
341 /// stdin is a terminal; required otherwise.
342 #[arg(long)]
343 input_file_type: Option<String>,
344
345 /// Compression level for a video input kind (`mov`/`m4v`/`mp4`):
346 /// `low`, `medium` (default), `high`, or `lossless` -- a small
347 /// vetted `libx265` CRF menu (ADR-0122). Ignored for an image input
348 /// kind. Interactively selected when omitted, stdin is a terminal,
349 /// and the input kind is video; defaults to `medium` otherwise.
350 #[arg(long)]
351 video_quality: Option<String>,
352
353 /// rclone source, e.g. `source:png/`. Interactively prompted when
354 /// omitted and stdin is a terminal; required otherwise.
355 #[arg(long)]
356 source_path: Option<String>,
357
358 /// rclone destination, e.g. `destination:jpg/`. Interactively
359 /// prompted when omitted and stdin is a terminal; required
360 /// otherwise.
361 #[arg(long)]
362 destination_path: Option<String>,
363
364 /// Local directory this run stages pulled source files, transcoded
365 /// results, and bookkeeping under. Defaults to a directory under
366 /// the OS temp directory when omitted.
367 #[arg(long)]
368 local_output: Option<PathBuf>,
369
370 /// Maximum number of files to transcode concurrently (the ffmpeg
371 /// phase only -- `rclone` manages its own internal parallelism via
372 /// `--download-transfers`/`--download-checkers`). Interactively
373 /// prompted (a cores-based default) when omitted and stdin is a
374 /// terminal; required otherwise.
375 #[arg(long)]
376 concurrency: Option<usize>,
377
378 /// rclone --transfers for the download (pull) phase. Defaults to 8
379 /// when omitted.
380 #[arg(long)]
381 download_transfers: Option<usize>,
382
383 /// rclone --checkers for the download (pull) phase. Defaults to 16
384 /// when omitted.
385 #[arg(long)]
386 download_checkers: Option<usize>,
387
388 /// Maximum number of concurrent per-file push (`rclone copyto`)
389 /// operations. Defaults to 8 when omitted. Separate from
390 /// `--download-transfers` to allow independent tuning of pull vs.
391 /// push phases (ADR-0129).
392 #[arg(long)]
393 upload_transfers: Option<usize>,
394
395 /// Maximum number of downloaded but not-yet-processed files
396 /// (pending transcode/push) before the pull phase is paused to allow
397 /// the pipeline to catch up. Defaults to `concurrency * 4` when
398 /// omitted. Enables automatic disk-pressure backpressure without
399 /// requiring manual tuning (ADR-0129).
400 #[arg(long)]
401 max_pending_downloads: Option<usize>,
402
403 /// rclone --tpslimit, used by the pull phase only. Unset by
404 /// default -- no rate cap.
405 #[arg(long)]
406 tpslimit: Option<usize>,
407
408 /// Alias of a configured bucket-config this run's report, the
409 /// shared observability log, and a transcript of its printed
410 /// output are uploaded to, always unencrypted, under a
411 /// `YYYY-MM-DD-job-name-{run-id}/` prefix (ADR-0100). Mandatory --
412 /// interactively selected when omitted and stdin is a terminal;
413 /// required otherwise.
414 #[arg(long)]
415 report_bucket: Option<String>,
416
417 /// Suppress every wizard prompt (ADR-0114): each omitted input
418 /// resolves via its non-interactive fallback (or errors if none
419 /// exists) instead of asking, and the final "proceed?"
420 /// confirmation is auto-answered yes.
421 #[arg(long)]
422 non_interactive: bool,
423 },
424}
425
426#[derive(Args, Debug)]
427pub struct RcloneArgs {
428 #[command(subcommand)]
429 pub action: RcloneAction,
430}
431
432#[derive(Subcommand, Debug)]
433pub enum RcloneAction {
434 /// Copies data from a configurable source to a configurable
435 /// destination by shelling out to `rclone copy`. `--source-path`/
436 /// `--destination-path` are raw `remote:path` strings passed straight
437 /// through to `rclone copy`'s argv, never pigeon `BucketConfig`/keyring
438 /// aliases (ADR-0101). `--transfers`/`--checkers`/`--tpslimit` are
439 /// overridable per-destination (ADR-0108); every other rclone
440 /// performance/retry flag stays fixed.
441 Copy {
442 /// rclone source, e.g. `source:media/`. Interactively prompted
443 /// when omitted and stdin is a terminal; required otherwise.
444 #[arg(long)]
445 source_path: Option<String>,
446
447 /// rclone destination, e.g. `destination:`. Interactively
448 /// prompted when omitted and stdin is a terminal; required
449 /// otherwise.
450 #[arg(long)]
451 destination_path: Option<String>,
452
453 /// Local directory this run's rclone log (also serving as this
454 /// job's report) and transcript are written under. Defaults to a
455 /// directory under the OS temp directory when omitted. Unlike
456 /// every other job, this is not a staging area for transferred
457 /// data -- rclone transfers directly source -> destination with no
458 /// pigeon-side staging.
459 #[arg(long)]
460 local_output: Option<PathBuf>,
461
462 /// Alias of a configured bucket-config this run's report (the
463 /// rclone log itself, for this job), the shared observability log,
464 /// and a transcript of its printed output are uploaded to, always
465 /// unencrypted, under a `YYYY-MM-DD-job-name-{run-id}/` prefix
466 /// (ADR-0100). Mandatory -- interactively selected when omitted
467 /// and stdin is a terminal; required otherwise.
468 #[arg(long)]
469 report_bucket: Option<String>,
470
471 /// rclone --transfers (concurrent file transfers). Defaults to 8
472 /// (ADR-0106) when omitted.
473 #[arg(long)]
474 transfers: Option<usize>,
475
476 /// rclone --checkers (concurrent list/compare operations).
477 /// Defaults to 16 (ADR-0106) when omitted.
478 #[arg(long)]
479 checkers: Option<usize>,
480
481 /// rclone --tpslimit (transactions/sec ceiling across transfers
482 /// and checkers combined). Unset by default -- no rate cap --
483 /// unless a specific destination needs one (ADR-0108).
484 #[arg(long)]
485 tpslimit: Option<usize>,
486
487 /// Suppress every wizard prompt (ADR-0114): each omitted input
488 /// resolves via its non-interactive fallback (or errors if none
489 /// exists) instead of asking, and the final "proceed?"
490 /// confirmation is auto-answered yes.
491 #[arg(long)]
492 non_interactive: bool,
493 },
494
495 /// Recursively and permanently deletes everything under a configurable
496 /// source path by shelling out to `rclone purge` (ADR-0110).
497 /// Irreversible -- there is no `--dry-run` and no undo beyond the
498 /// existing `--non-interactive`/interactive confirmation. Unlike
499 /// `copy`, there is no destination and no per-file transfer-rate
500 /// tuning, since purge transfers no file content.
501 Delete {
502 /// rclone source to recursively and permanently delete, e.g.
503 /// `source:media/`. Interactively prompted when omitted and stdin
504 /// is a terminal; required otherwise.
505 #[arg(long)]
506 source_path: Option<String>,
507
508 /// Local directory this run's rclone log (also serving as this
509 /// job's report) and transcript are written under. Defaults to a
510 /// directory under the OS temp directory when omitted.
511 #[arg(long)]
512 local_output: Option<PathBuf>,
513
514 /// Alias of a configured bucket-config this run's report, the
515 /// shared observability log, and a transcript of its printed
516 /// output are uploaded to, always unencrypted, under a
517 /// `YYYY-MM-DD-job-name-{run-id}/` prefix (ADR-0100). Mandatory --
518 /// interactively selected when omitted and stdin is a terminal;
519 /// required otherwise.
520 #[arg(long)]
521 report_bucket: Option<String>,
522
523 /// rclone --checkers (concurrent list operations during
524 /// enumeration). Defaults to 16 when omitted.
525 #[arg(long)]
526 checkers: Option<usize>,
527
528 /// Suppress every wizard prompt (ADR-0114): each omitted input
529 /// resolves via its non-interactive fallback (or errors if none
530 /// exists) instead of asking, and the final "proceed?"
531 /// confirmation is auto-answered yes.
532 #[arg(long)]
533 non_interactive: bool,
534 },
535}
536
537impl Observable for JobType {
538 fn command_name(&self) -> &'static str {
539 match self {
540 JobType::EmailSync { .. } => "job.email-sync",
541 JobType::DecryptFiles { .. } => "job.decrypt-files",
542 JobType::EmailPull { .. } => "job.email-pull",
543 JobType::Deduplicate { .. } => "job.deduplicate",
544 JobType::Rclone(args) => match args.action {
545 RcloneAction::Copy { .. } => "job.rclone-copy",
546 RcloneAction::Delete { .. } => "job.rclone-delete",
547 },
548 JobType::Transform { .. } => "job.transform",
549 }
550 }
551}
552
553impl JobType {
554 /// The bare dash-form job name (e.g. `"deduplicate"`) used in the
555 /// report-bucket upload prefix (ADR-0100) -- `command_name()`'s value
556 /// with its `"job."` prefix stripped, rather than a 6th place in this
557 /// codebase repeating the same job-name literals already duplicated
558 /// across every job's `worker.rs` metrics call sites.
559 pub(crate) fn job_name(&self) -> &'static str {
560 self.command_name().trim_start_matches("job.")
561 }
562}