pigeon-cli 0.4.0

Pigeon: authenticate, sink, and transform personal data from external services.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74

//! `pigeon job run deduplicate` (ADR-0082): recursively pulls every object from a
//! source bucket, always inflates every zip encountered (containers
//! themselves never uploaded, only their inflated contents), content-hashes
//! (SHA-256) everything bucket-wide to keep one byte-identical copy of each
//! file, writes a human-readable merge report, and optionally uploads the
//! result unencrypted to a (possibly different) bucket-config.
//!
//! Two directory trees exist per run, mirrored under `local_output`:
//! `local_output/.staging/` holds bookkeeping only (`.processed` checkpoint,
//! `.content-hashes` index, raw downloaded/extracted files awaiting hash and
//! placement) and is never uploaded; `local_output/result/<extension>/...`
//! holds the real deliverable and is what the upload phase walks.
//! `local_output/deduplicate-report.txt` sits at the top level, deliberately
//! outside `result/`, so it can never be swept into the destination bucket
//! (`core::data::collect_files` does not skip dotfiles/dot-directories, so
//! this split is load-bearing, not cosmetic -- see `dedup.rs`).

pub mod dedup;
mod manifest;
pub mod wizard;
mod worker;

use std::path::PathBuf;

use crate::commands::keyring::bucket::store::BucketConfig;
use crate::core::job::Job;

pub(crate) use manifest::{DeduplicatePlan, gather_pending};
pub(crate) use worker::DeduplicateSummary;

/// No `encryptor` field at all -- a permanent scoping decision (ADR-0082
/// ยง0), not a temporarily-unused slot.
pub(crate) struct DeduplicateJob {
    pub source_bucket: BucketConfig,
    pub source_secret: String,
    pub local_output: PathBuf,
    pub remote: Option<(BucketConfig, String)>,
}

impl Job for DeduplicateJob {
    type Plan = DeduplicatePlan;
    type Summary = DeduplicateSummary;

    async fn gather(&self) -> Result<DeduplicatePlan, String> {
        gather_pending(
            &self.source_bucket,
            &self.source_secret,
            &self.local_output.join(".staging"),
        )
        .await
    }

    async fn run(
        self,
        plan: DeduplicatePlan,
        concurrency: usize,
        upload_concurrency: usize,
    ) -> Result<DeduplicateSummary, String> {
        let remote_ref = self
            .remote
            .as_ref()
            .map(|(bucket_config, secret)| (bucket_config, secret.as_str()));
        worker::run_deduplicate_job(
            &self.source_bucket,
            &self.source_secret,
            &self.local_output,
            plan.tasks,
            concurrency,
            upload_concurrency,
            remote_ref,
        )
        .await
    }
}