crate-activity 0.7.0

This crate provides a way to monitor the usage for a set of crates.io crates
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117

crate::ix!();

#[derive(Debug, Getters)]
pub struct CrateActivityData {

    #[getset(get = "pub")]
    summaries: Vec<CrateUsageSummary>,
    
    #[getset(get = "pub")]
    interval_downloads_1d: HashMap<String, i64>,
    
    #[getset(get = "pub")]
    interval_downloads_3d: HashMap<String, i64>,

    #[getset(get = "pub")]
    interval_downloads_7d: HashMap<String, i64>,
}

pub async fn gather_crate_activity_data(
    ignore_cache:   bool,
    crate_names:    &[String],
    user_agent:     &str,
    config_dir:     &Path,
    one_day_ago:    NaiveDate,
    three_days_ago: NaiveDate,
    seven_days_ago: NaiveDate,
) -> Result<CrateActivityData, CrateActivityError> {
    use futures::{StreamExt};

    println!(
        "Gathering crate activity data for {} crates (ignore_cache={})",
        crate_names.len(),
        ignore_cache
    );

    // We'll limit concurrency to avoid overwhelming crates.io.
    let concurrency_limit = 8usize;

    // Create a stream of futures (one for each crate).
    let crate_fetches = futures::stream::iter(crate_names.iter().map(|crate_name| {
        let crate_name = crate_name.clone();
        let ua = user_agent.to_string();
        let cfg_dir = config_dir.to_path_buf();
        async move {
            debug!("Fetching usage for crate '{}'", crate_name);
            match fetch_usage(ignore_cache, &ua, &cfg_dir, &crate_name).await {
                Ok(Some(response)) => {
                    debug!("Successfully fetched usage for crate '{}'", crate_name);
                    Some((crate_name, response))
                },
                Ok(None) => {
                    warn!("No data for crate '{}'", crate_name);
                    None
                },
                Err(e) => {
                    error!("Failed to fetch data for '{}': {:?}", crate_name, e);
                    None
                }
            }
        }
    }))
    .buffer_unordered(concurrency_limit);

    // Collect all results in parallel.
    let results: Vec<Option<(String, CrateResponse)>> = crate_fetches.collect().await;

    let mut summaries             = Vec::new();
    let mut interval_downloads_1d = HashMap::new();
    let mut interval_downloads_3d = HashMap::new();
    let mut interval_downloads_7d = HashMap::new();

    // Process the completed fetches.
    for item in results {
        if let Some((crate_name, response)) = item {
            let summary = analyze_usage(&crate_name, response.version_downloads().to_vec());
            summaries.push(summary);

            let downloads_last_1d: i64 = response
                .version_downloads()
                .iter()
                .filter(|d| *d.date() >= one_day_ago)
                .map(|d| d.downloads())
                .sum();

            let downloads_last_3d: i64 = response
                .version_downloads()
                .iter()
                .filter(|d| *d.date() >= three_days_ago)
                .map(|d| d.downloads())
                .sum();

            let downloads_last_7d: i64 = response
                .version_downloads()
                .iter()
                .filter(|d| *d.date() >= seven_days_ago)
                .map(|d| d.downloads())
                .sum();

            interval_downloads_1d.insert(crate_name.clone(), downloads_last_1d);
            interval_downloads_3d.insert(crate_name.clone(), downloads_last_3d);
            interval_downloads_7d.insert(crate_name.clone(), downloads_last_7d);
        }
    }

    println!(
        "Collected activity data for {} crates (out of {} requested).",
        summaries.len(),
        crate_names.len()
    );

    Ok(CrateActivityData {
        summaries,
        interval_downloads_1d,
        interval_downloads_3d,
        interval_downloads_7d,
    })
}