pps/
lib.rs

1use std::fmt;
2use chrono::prelude::*;
3use std::convert::From;
4use lazy_static::lazy_static;
5use scraper::{Html, Selector, ElementRef};
6use tokio::sync::mpsc;
7use tokio::process::Command;
8use anyhow::Result;
9use tabled::Tabled;
10use serde::{Serialize, Deserialize};
11use thousands::Separable;
12use backoff::ExponentialBackoff;
13use backoff::future::retry;
14use std::cmp::Ordering;
15
16lazy_static! {
17    static ref NAME_SELECTOR: Selector = Selector::parse("span.package-snippet__name").unwrap();
18    static ref VERSION_SELECTOR: Selector = Selector::parse("span.package-snippet__version").unwrap();
19    static ref RELEASE_SELECTOR: Selector = Selector::parse("span.package-snippet__released").unwrap();
20    static ref DESCRIPTION_SELECTOR: Selector = Selector::parse("p.package-snippet__description").unwrap();
21    static ref DATETIME_SELECTOR: Selector = Selector::parse("time").unwrap();
22}
23
24fn unwrap_selector(input: &ElementRef, selector: &Selector) -> String {
25    input.select(selector).next().map(|e| e.inner_html()).unwrap_or("".into())
26}
27
28fn unwrap_time_selector(input: &ElementRef) -> Option<DateTime<Utc>> {
29    input.select(&RELEASE_SELECTOR).next()
30        .and_then(|release| release.select(&DATETIME_SELECTOR).next())
31        .and_then(|time| time.value().attr("datetime"))
32        .and_then(|dt| dt.parse::<DateTime<Utc>>().ok())
33}
34
35fn format_date(release: &DateTime<Utc>) -> String {
36    release.format("%Y-%m-%d").to_string()
37}
38
39#[derive(Serialize,Deserialize,Debug,PartialEq,Eq)]
40pub struct Downloads {
41    last_day: u64,
42    last_week: u64,
43    last_month: u64,
44}
45
46impl fmt::Display for Downloads {
47    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
48        write!(f, "{} / {} / {}", 
49            self.last_day.separate_with_commas(), 
50            self.last_week.separate_with_commas(), 
51            self.last_month.separate_with_commas()
52            )
53    }
54}
55
56impl Ord for Downloads {
57    fn cmp(&self, other: &Self) -> Ordering {
58        self.last_month.cmp(&other.last_month)
59    }
60}
61
62impl PartialOrd for Downloads {
63    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
64        Some(self.cmp(other))
65    }
66}
67
68
69#[derive(Serialize,Deserialize)]
70struct DownloadsResponse {
71    data: Downloads,
72    package: String,
73    #[serde(rename(deserialize = "type", serialize = "type"))]
74    response_type: String,
75}
76
77
78#[derive(Debug, PartialEq)]
79pub struct LocalPackage {
80    pub name: String,
81    pub version: String,
82}
83
84impl LocalPackage {
85    fn from_str(input: &str) -> Result<LocalPackage> {
86        let fields: Vec<&str> = input.split_whitespace().collect();
87        Ok(LocalPackage{
88            name: fields[0].into(), 
89            version: fields[1].into()
90        })
91    }
92}
93
94fn display_installed(possible_version: &Option<String>) -> String {
95    possible_version
96        .as_ref()
97        .unwrap_or(&String::from(""))
98        .into()
99}
100
101fn display_downloads(possible_downloads: &Option<Downloads>) -> String {
102    match possible_downloads {
103        Some(d) => format!("{}", d),
104        None => "".into()
105    }
106}
107
108#[derive(Debug,Tabled)]
109pub struct Package {
110    #[header("Name")]
111    pub name: String,
112    #[header("Installed")]
113    #[field(display_with="display_installed")]
114    pub installed: Option<String>,
115    #[header("Version")]
116    pub version: String,
117    #[header("Released")]
118    #[field(display_with="format_date")]
119    pub release: DateTime<Utc>,
120    #[header("Description")]
121    pub description: String,
122    #[header("Downloads")]
123    #[field(display_with="display_downloads")]
124    pub downloads: Option<Downloads>,
125}
126
127impl From<&ElementRef<'_>> for Package {
128    fn from(input: &ElementRef) -> Self {
129        let release = unwrap_time_selector(input);
130        Package{
131            name: unwrap_selector(input, &NAME_SELECTOR),
132            installed: None,
133            version: unwrap_selector(input, &VERSION_SELECTOR),
134            release: release.unwrap(),
135            description: unwrap_selector(input, &DESCRIPTION_SELECTOR),
136            downloads: None,
137        }
138    }
139
140}
141
142async fn get_with_retry(url: &str) -> Result<String> {
143     Ok(retry(ExponentialBackoff::default(), || async {
144            let body = reqwest::get(url)
145                .await?
146                .error_for_status()?
147                .text()
148                .await?;
149            Ok(body)
150        }).await?)
151}
152
153impl Package {
154    pub fn local(&mut self, version: &str) {
155        self.installed = Some(version.into())
156    }
157
158    pub async fn update_downloads(&mut self) {
159
160        let url = format!("https://pypistats.org/api/packages/{}/recent", self.name.to_lowercase());
161   
162        let body: String = get_with_retry(&url).await.unwrap();
163        let data: DownloadsResponse = serde_json::from_str(&body).unwrap();
164        self.downloads = Some(data.data);
165    }
166}
167
168
169pub async fn query_pypi(name: String, pages: usize) -> Result<Vec<Package>>{
170    let client = reqwest::Client::new();
171    let (tx, mut rx) = mpsc::channel(32);
172
173    let package_snippet = Selector::parse("a.package-snippet").unwrap();
174
175    tokio::spawn(async move {
176        for page_idx in (1..=pages).map(|i| i.to_string()) {
177            let query_params = vec![("q", &name), ("page", &page_idx)];
178
179            let page_body = client.get("https://pypi.org/search/")
180                .query(&query_params)
181                .send()
182                .await;
183            tx.send(page_body).await.expect("can send on package channel");
184        }
185    });
186
187    let mut packages = vec![];
188
189    while let Some(response) = rx.recv().await {
190        let page_body = response?.text().await?;
191        let page_result = Html::parse_document(&page_body);
192        for element in page_result.select(&package_snippet) {
193            let package = Package::from(&element);
194            packages.push(package);
195        }
196    }
197
198    Ok(packages)
199}
200
201pub async fn get_installed_packages() -> Result<Vec<LocalPackage>> {
202    let output = Command::new("pip")
203        .arg("list")
204        .arg("installed")
205        .output()
206        .await?;
207
208    let stdout: String = String::from_utf8(output.stdout)?;
209
210    let mut out = vec![];
211    for line in stdout.lines().skip(2) {
212        out.push(LocalPackage::from_str(line)?)
213    }
214    Ok(out)
215}
216
217pub async fn get_downloads(package_name: &str) -> Result<Downloads> {
218    let url = format!("https://pypistats.org/api/packages/{}/recent", package_name);
219    let data: DownloadsResponse = reqwest::get(&url)
220        .await?
221        .json()
222        .await?;
223    Ok(data.data)
224}
225
226#[cfg(test)]
227mod tests {
228    use super::*;
229
230    #[test]
231    fn parse_package_data_test() {
232        let input = r#"
233            <a class="package-snippet" href="/project/gitlab3/">
234            <h3 class="package-snippet__title">
235              <span class="package-snippet__name">gitlab3</span>
236              <span class="package-snippet__version">0.5.8</span>
237              <span class="package-snippet__released"><time datetime="2017-03-18T19:38:52+0000" data-controller="localized-time" data-localized-time-relative="true" data-localized-time-show-time="false" title="2017-03-18 20:38:52" aria-label="2017-03-18 20:38:52">Mar 18, 2017</time></span>
238            </h3>
239            <p class="package-snippet__description">GitLab API v3 Python Wrapper.</p>
240          </a>"#;
241        let page = Html::parse_fragment(input);
242        let snippet = page.root_element();
243        let package = Package::from(&snippet);
244
245        assert_eq!(package.name, "gitlab3");
246        assert_eq!(package.version, "0.5.8");
247        assert_eq!(package.release, "2017-03-18T19:38:52+0000".parse::<DateTime<Utc>>().unwrap());
248        assert_eq!(package.description, "GitLab API v3 Python Wrapper.");
249    }
250
251    #[tokio::test]
252    async fn get_installed_packages_test() {
253       let installed_packages = get_installed_packages().await.unwrap();
254       let pip_package = LocalPackage{ name: "pip".into(), version: "21.1.2".into() };
255       assert!(installed_packages.contains(&pip_package));
256    }
257
258    #[tokio::test]
259    async fn get_downloads_test() {
260        let package_name = "python-gitlab";
261        let downloads = get_downloads(package_name).await.unwrap();
262        assert!(downloads.last_month > 0);
263    }
264}
pps/lib.rs

pps/
lib.rs
