trendkit_core/
cratesio.rs

1//! crates.io source: most-downloaded crates, snapshotted over time.
2//!
3//! Download totals are cumulative, so `Δdownloads/day` is the trend
4//! signal. The list endpoint is used (no auth); crates.io policy
5//! requires an identifying User-Agent, which fetch-kit sets.
6
7use std::time::Duration;
8
9use fetch_kit::Client;
10use serde::{Deserialize, Serialize};
11
12use crate::clock::unix_seconds;
13use crate::error::Result;
14use crate::model::{Observation, SourceKind, SubjectRecord};
15use crate::source::{CollectCtx, CollectReport, TrendSource};
16
17const CRATES_API: &str = "https://crates.io/api/v1";
18const USER_AGENT: &str = "trendkit/0.2 (https://github.com/WyattAu/trendkit)";
19const PER_PAGE: u32 = 100;
20/// crates.io asks for <= 1 req/s from identifiable clients.
21pub const CRATES_PACING: Duration = Duration::from_millis(1_200);
22
23/// crates.io trend source.
24#[derive(Debug, Clone)]
25pub struct CratesSource {
26    client: Client,
27}
28
29impl CratesSource {
30    /// Build a source.
31    #[must_use]
32    pub fn new() -> CratesSource {
33        CratesSource {
34            client: Client::builder().user_agent(USER_AGENT).build(),
35        }
36    }
37}
38
39impl Default for CratesSource {
40    fn default() -> Self {
41        CratesSource::new()
42    }
43}
44
45impl TrendSource for CratesSource {
46    fn name(&self) -> &'static str {
47        "cratesio"
48    }
49
50    fn collect(
51        &self,
52        ctx: &CollectCtx,
53    ) -> impl std::future::Future<Output = Result<CollectReport>> {
54        let ctx = ctx.clone();
55        async move {
56            let now = unix_seconds(ctx.clock.as_ref());
57            let mut total = 0usize;
58            for page in 1..=ctx.pages {
59                let params = CratesQuery {
60                    sort: "downloads",
61                    per_page: PER_PAGE,
62                    page,
63                };
64                let url = format!("{CRATES_API}/crates");
65                let response: CratesResponse =
66                    self.client.get(url).query(&params).json_response().await?;
67                let count = response.crates.len();
68                for krate in response.crates {
69                    let observation = Observation {
70                        ts: now,
71                        score: krate.downloads,
72                        extra1: krate.recent_downloads.map(|f| f as i64).unwrap_or(0),
73                        extra2: 0,
74                    };
75                    let subject = SubjectRecord::from(krate);
76                    let subject_id = ctx.store.upsert_subject(&subject)?;
77                    ctx.store.insert_observation(subject_id, &observation)?;
78                    total += 1;
79                }
80                if count < PER_PAGE as usize || page == ctx.pages {
81                    break;
82                }
83                tokio::time::sleep(CRATES_PACING).await;
84            }
85            ctx.store
86                .record_run(now, self.name(), i64::from(ctx.pages), total as i64)?;
87            Ok(CollectReport {
88                source: self.name(),
89                items: total,
90                pages: ctx.pages as usize,
91            })
92        }
93    }
94}
95
96#[derive(Debug, Serialize)]
97struct CratesQuery<'a> {
98    sort: &'a str,
99    per_page: u32,
100    page: u32,
101}
102
103#[derive(Debug, Deserialize)]
104struct CratesResponse {
105    #[serde(default)]
106    crates: Vec<CrateJson>,
107}
108
109/// The subset of a crates.io crate summary trendkit consumes.
110#[derive(Debug, Clone, Deserialize)]
111#[allow(dead_code)] // payload-derived; kept for payload fidelity and debugging
112pub(crate) struct CrateJson {
113    #[serde(default)]
114    pub(crate) name: String,
115    #[serde(default)]
116    pub(crate) description: Option<String>,
117    #[serde(default)]
118    pub(crate) downloads: i64,
119    #[serde(default)]
120    pub(crate) recent_downloads: Option<f64>,
121    #[serde(default)]
122    pub(crate) max_version: Option<String>,
123    #[serde(default)]
124    pub(crate) created_at: Option<String>,
125    #[serde(default)]
126    pub(crate) keywords: Option<Vec<String>>,
127}
128
129impl From<CrateJson> for SubjectRecord {
130    fn from(json: CrateJson) -> Self {
131        let url = format!("https://crates.io/crates/{}", json.name);
132        SubjectRecord {
133            kind: SourceKind::Crate,
134            key: json.name,
135            title: json.description,
136            lang: None,
137            url: Some(url),
138            topics: json.keywords.unwrap_or_default(),
139            created_at: json.created_at,
140        }
141    }
142}
143
144/// Parse a crates.io list response body into subjects.
145///
146/// # Errors
147/// Fails when the payload is not a crates.io response.
148pub fn parse_crates_json(payload: &str) -> Result<Vec<SubjectRecord>> {
149    let response: CratesResponse = serde_json::from_str(payload)
150        .map_err(|e| crate::error::TrendError::Config(format!("invalid crates payload: {e}")))?;
151    Ok(response
152        .crates
153        .into_iter()
154        .map(SubjectRecord::from)
155        .collect())
156}
157
158#[cfg(test)]
159mod tests {
160    #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
161
162    use super::*;
163
164    const SAMPLE: &str = r#"{
165        "crates": [
166            {
167                "name": "serde",
168                "description": "A serialization framework",
169                "downloads": 300000000,
170                "recent_downloads": 12345678.0,
171                "max_version": "1.0.220",
172                "created_at": "2016-01-05T00:00:00Z",
173                "keywords": ["serialization", "no_std"]
174            },
175            {
176                "name": "bare-crate",
177                "downloads": 99
178            }
179        ],
180        "meta": { "total": 2 }
181    }"#;
182
183    #[test]
184    fn parse_extracts_crates_with_defaults() {
185        let subjects = parse_crates_json(SAMPLE).unwrap();
186        assert_eq!(subjects.len(), 2);
187        assert_eq!(subjects[0].kind, SourceKind::Crate);
188        assert_eq!(subjects[0].key, "serde");
189        assert_eq!(
190            subjects[0].title.as_deref(),
191            Some("A serialization framework")
192        );
193        assert_eq!(
194            subjects[0].topics,
195            vec!["serialization".to_owned(), "no_std".to_owned()]
196        );
197        // Missing fields default cleanly.
198        assert_eq!(subjects[1].topics, Vec::<String>::new());
199        assert_eq!(subjects[1].title, None);
200    }
201
202    #[test]
203    fn parse_rejects_garbage() {
204        assert!(parse_crates_json("<html>").is_err());
205    }
206}
trendkit_core/cratesio.rs

trendkit_core/
cratesio.rs
