1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
//! Client and methods for retrieving information from the crates.io API
//!
//! _Note_: Due to the crates.io crawler policy, the amount of requests that
//! can be made is limited. [`CratesIoClient`] attempts to make this less
//! noticeable with caching and doing large fetches, but please keep this in
//! mind.
//!
//! See [the crates.io crawler policy](https://crates.io/policies#crawlers) for
//! more information.
use std::{collections::HashMap, time::Duration};
use crates_io_api::{Crate, CrateResponse, SyncClient, Version};
use crate::NameVersion;
/// Wrapper around a [`crates_io_api::SyncClient`], with added caching
pub struct CratesIoClient {
client: SyncClient,
/// Cache between crate name and downloads info
///
/// We do not want requests for the same crate to fail and then work in the
/// same query, so we store if we were able to find it the first time via
/// the option.
cache: HashMap<String, Option<CrateResponse>>,
}
impl CratesIoClient {
/// Creates a new `crates.io` client and cache
///
/// # Panics
///
/// Panics if the given agent parameters are invalid.
#[must_use]
pub fn new(user_agent: &str, rate_limit: Duration) -> Self {
let client =
SyncClient::new(user_agent, rate_limit).unwrap_or_else(|e| {
panic!("could not create CratesIoClient due to error: {e}");
});
Self {
client,
cache: HashMap::new(),
}
}
/// Retrieves information about a crate from the `crates.io` API
///
/// Will return `None` if the request fails, and will cache this crate as
/// such.
pub fn crate_response(
&mut self,
crate_name: &str,
) -> Option<&mut CrateResponse> {
self.cache.entry(crate_name.to_string()).or_insert_with(|| {
match self.client.get_crate(crate_name) {
Ok(cr) => Some(cr),
Err(e) => {
eprintln!("failed to retrieve crates.io information about {crate_name} due to error: {e}");
None
}
}
}).as_mut()
}
/// Retrieve data about a crate from the `crates.io` API
pub fn crate_data(&mut self, crate_name: &str) -> Option<&Crate> {
self.crate_response(crate_name).map(|cr| &cr.crate_data)
}
/// Retrieves data about all versions of a crate from the `crates.io` API
pub fn versions(&mut self, crate_name: &str) -> Option<&Vec<Version>> {
self.crate_response(crate_name).map(|cr| &cr.versions)
}
/// Returns the number of versions of a crate from the `crates.io` API
pub fn versions_count(&mut self, crate_name: &str) -> Option<usize> {
self.versions(crate_name).map(Vec::len)
}
/// Retrieves the total amount of downloads for a crate, all versions
pub fn total_downloads(&mut self, crate_name: &str) -> Option<u64> {
self.crate_data(crate_name).map(|c| c.downloads)
}
/// Retrieves the total amount of downloads for a crate, all versions
pub fn recent_downloads(&mut self, crate_name: &str) -> Option<u64> {
self.crate_data(crate_name).and_then(|c| c.recent_downloads)
}
/// Retrieves the total amount of downloads for a specific crate version
pub fn version_downloads(
&mut self,
name_version: &NameVersion,
) -> Option<u64> {
self.versions(&name_version.name).and_then(|versions| {
versions.iter().find_map(|v| {
match rustsec::Version::parse(&v.num) {
Ok(current_version) => {
if current_version == name_version.version {
Some(v.downloads)
} else {
None
}
}
Err(e) => {
eprintln!("could not parse crates.io version for {name_version:?} due to error: {e}");
None
}
} }) })
}
/// Returns if this version is yanked from `crates.io`
pub fn yanked(&mut self, name_version: &NameVersion) -> Option<bool> {
self.versions(&name_version.name).and_then(|versions| {
versions.iter().find_map(|v| {
match rustsec::Version::parse(&v.num) {
Ok(current_version) => {
if current_version == name_version.version {
Some(v.yanked)
} else {
None
}
}
Err(e) => {
eprintln!("could not parse crates.io version for {name_version:?} due to error: {e}");
None
}
}
})
})
}
/// Retrieves all versions for a crate that has been marked as yanked
///
/// If only the count of yanked versions is desired, use
/// [`yanked_versions_count`](Self::yanked_versions_count) instead.
pub fn yanked_versions(&mut self, crate_name: &str) -> Option<Vec<String>> {
self.versions(crate_name).map(|versions| {
versions
.iter()
.filter_map(|fv| {
if fv.yanked {
// We do not parse version, as that may fail, leading
// to odd results
Some(fv.num.clone())
} else {
None
}
})
.collect()
})
}
/// Counts the number of versions marked as _yanked_ on `crates.io` for this
/// crate
pub fn yanked_versions_count(&mut self, crate_name: &str) -> Option<usize> {
// Do not rely on Self::yanked_version, as it is more expensive
self.versions(crate_name)
.map(|versions| versions.iter().filter(|v| v.yanked).count())
}
/// Calculates the ratio of yanked versions to all crate versions
pub fn yanked_ratio(&mut self, crate_name: &str) -> Option<f64> {
self.yanked_versions_count(crate_name).and_then(|y| {
self.versions_count(crate_name).map(|v| y as f64 / v as f64)
})
}
}
impl Default for CratesIoClient {
fn default() -> Self {
let user_agent = std::env::var("USER_AGENT")
.expect("USER_AGENT environment variable not set");
Self::new(&user_agent, Duration::from_secs(1))
}
}