Skip to main content

docsrs_cli/
crates_io.rs

1//! crates.io search-crates operation.
2
3use serde::{Deserialize, Serialize};
4use url::Url;
5
6use crate::config::{HOST_CRATES_IO, MAX_PER_PAGE};
7use crate::error::{AppError, AppResult, ErrorKind};
8use crate::http::{HttpClient, content_type_looks_json, decode_utf8};
9
10/// One hit from crates.io search.
11#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
12pub struct CrateSearchHit {
13    /// Crate name on crates.io.
14    pub name: String,
15    /// Short crate description (may be empty).
16    pub description: String,
17    /// Total download count.
18    pub downloads: u64,
19    /// Newest or default version string for display.
20    pub version: String,
21    /// Documentation URL when provided by crates.io.
22    #[serde(skip_serializing_if = "Option::is_none")]
23    pub documentation: Option<String>,
24    /// Highest published version when present.
25    #[serde(skip_serializing_if = "Option::is_none")]
26    pub max_version: Option<String>,
27    /// Highest stable version when present.
28    #[serde(skip_serializing_if = "Option::is_none")]
29    pub max_stable_version: Option<String>,
30    /// Default version selected by crates.io.
31    #[serde(skip_serializing_if = "Option::is_none")]
32    pub default_version: Option<String>,
33    /// Recent download count when present.
34    #[serde(skip_serializing_if = "Option::is_none")]
35    pub recent_downloads: Option<u64>,
36    /// Whether the query matched the crate name exactly.
37    #[serde(skip_serializing_if = "Option::is_none")]
38    pub exact_match: Option<bool>,
39    /// Whether the default version is yanked.
40    #[serde(skip_serializing_if = "Option::is_none")]
41    pub yanked: Option<bool>,
42    /// Source repository URL when present.
43    #[serde(skip_serializing_if = "Option::is_none")]
44    pub repository: Option<String>,
45    /// Project homepage URL when present.
46    #[serde(skip_serializing_if = "Option::is_none")]
47    pub homepage: Option<String>,
48}
49
50/// Pagination meta from crates.io (may contain `page=` or `seek=` tokens).
51#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
52pub struct SearchMeta {
53    /// Total hits reported by crates.io.
54    pub total: u64,
55    /// Opaque next-page token when more results exist.
56    #[serde(skip_serializing_if = "Option::is_none")]
57    pub next_page: Option<String>,
58    /// Opaque previous-page token when available.
59    #[serde(skip_serializing_if = "Option::is_none")]
60    pub prev_page: Option<String>,
61}
62
63/// Full search-crates data payload.
64#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
65pub struct SearchCratesData {
66    /// Original search query string.
67    pub query: String,
68    /// 1-based page number used for the request.
69    pub page: u32,
70    /// Page size used for the request.
71    pub per_page: u32,
72    /// Sort order API token.
73    pub sort: String,
74    /// Search hits on this page.
75    pub hits: Vec<CrateSearchHit>,
76    /// Pagination metadata from crates.io.
77    pub meta: SearchMeta,
78    /// True when the HTTP body was served from the local disk cache.
79    pub cache_hit: bool,
80}
81
82#[derive(Debug, Deserialize)]
83struct ApiResponse {
84    crates: Vec<ApiCrate>,
85    meta: ApiMeta,
86}
87
88#[derive(Debug, Deserialize)]
89struct ApiCrate {
90    name: String,
91    description: Option<String>,
92    downloads: u64,
93    newest_version: Option<String>,
94    max_version: Option<String>,
95    max_stable_version: Option<String>,
96    default_version: Option<String>,
97    documentation: Option<String>,
98    recent_downloads: Option<u64>,
99    exact_match: Option<bool>,
100    yanked: Option<bool>,
101    repository: Option<String>,
102    homepage: Option<String>,
103}
104
105#[derive(Debug, Deserialize)]
106struct ApiMeta {
107    total: u64,
108    next_page: Option<String>,
109    prev_page: Option<String>,
110}
111
112/// Clamp crates.io pagination to product bounds (`per_page` 1..=100, `page` >= 1).
113///
114/// Dry-run `planned_params` and the request URL MUST use these same values so
115/// agents never see `page=0` in params while the URL has `page=1`.
116pub fn clamp_search_pagination(per_page: u32, page: u32) -> (u32, u32) {
117    (per_page.clamp(1, MAX_PER_PAGE), page.max(1))
118}
119
120/// Build the planned crates.io search URL.
121///
122/// # Errors
123///
124/// Returns [`ErrorKind::Internal`] when the base URL cannot be parsed.
125pub fn planned_url(query: &str, per_page: u32, sort: &str, page: u32) -> AppResult<Url> {
126    planned_url_on_host(HOST_CRATES_IO, query, per_page, sort, page)
127}
128
129/// Build search URL against an arbitrary host (used by tests / local mocks).
130///
131/// # Errors
132///
133/// Returns [`ErrorKind::Internal`] when `host` does not form a valid base URL.
134pub fn planned_url_on_host(
135    host: &str,
136    query: &str,
137    per_page: u32,
138    sort: &str,
139    page: u32,
140) -> AppResult<Url> {
141    let (per_page, page) = clamp_search_pagination(per_page, page);
142    let scheme = if host.starts_with("http://") || host.starts_with("https://") {
143        ""
144    } else {
145        "https://"
146    };
147    let base = if scheme.is_empty() {
148        format!("{host}/api/v1/crates")
149    } else {
150        format!("{scheme}{host}/api/v1/crates")
151    };
152    let mut url = Url::parse(&base)
153        .map_err(|e| AppError::with_source(ErrorKind::Internal, "invalid crates.io base URL", e))?;
154    {
155        let mut q = url.query_pairs_mut();
156        q.append_pair("q", query);
157        q.append_pair("per_page", &per_page.to_string());
158        q.append_pair("sort", sort);
159        q.append_pair("page", &page.to_string());
160    }
161    Ok(url)
162}
163
164
165/// Build search URL using an opaque `next_page` / `prev_page` token from crates.io meta.
166///
167/// Accepts:
168/// - full query string (`?q=…&page=2` or `q=…&page=2`)
169/// - pure page number string
170/// - `seek=…` tokens
171///
172/// # Errors
173///
174/// Returns [`ErrorKind::Internal`] when the base URL is invalid.
175/// Returns [`ErrorKind::InvalidInput`] when the token cannot form a valid URL.
176pub fn planned_url_with_page_token(
177    host: &str,
178    query: &str,
179    per_page: u32,
180    sort: &str,
181    page_token: &str,
182) -> AppResult<Url> {
183    let token = page_token.trim();
184    if token.is_empty() {
185        return Err(AppError::new(
186            ErrorKind::InvalidInput,
187            "page token is empty",
188        ));
189    }
190    // Pure numeric → page number
191    if let Ok(page) = token.parse::<u32>() {
192        return planned_url_on_host(host, query, per_page, sort, page);
193    }
194    let scheme = if host.starts_with("http://") || host.starts_with("https://") {
195        ""
196    } else {
197        "https://"
198    };
199    let base = if scheme.is_empty() {
200        format!("{host}/api/v1/crates")
201    } else {
202        format!("{scheme}{host}/api/v1/crates")
203    };
204    let base_url = Url::parse(&base)
205        .map_err(|e| AppError::with_source(ErrorKind::Internal, "invalid crates.io base URL", e))?;
206
207    let qs = token.trim_start_matches('?');
208    // If token already carries a full query, use it as the request query.
209    if qs.contains('=') {
210        let mut url = base_url;
211        url.set_query(Some(qs));
212        return Ok(url);
213    }
214    // Opaque seek-like token
215    let (per_page, _) = clamp_search_pagination(per_page, 1);
216    let mut url = base_url;
217    {
218        let mut q = url.query_pairs_mut();
219        q.append_pair("q", query);
220        q.append_pair("per_page", &per_page.to_string());
221        q.append_pair("sort", sort);
222        q.append_pair("seek", qs);
223    }
224    Ok(url)
225}
226/// Parse crates.io JSON body into product types (offline-testable).
227///
228/// # Errors
229///
230/// Returns [`ErrorKind::Parse`] when `text` is not valid crates.io search JSON.
231pub fn parse_search_body(
232    text: &str,
233    query: &str,
234    page: u32,
235    per_page: u32,
236    sort: &str,
237    cache_hit: bool,
238) -> AppResult<SearchCratesData> {
239    let (per_page, page) = clamp_search_pagination(per_page, page);
240    let parsed: ApiResponse = serde_json::from_str(text).map_err(|e| {
241        AppError::with_source(ErrorKind::Parse, "failed to parse crates.io JSON", e)
242    })?;
243
244    let hits = parsed
245        .crates
246        .into_iter()
247        .map(|c| CrateSearchHit {
248            name: c.name,
249            description: c
250                .description
251                .filter(|s| !s.is_empty())
252                .unwrap_or_else(|| "No description available".to_string()),
253            downloads: c.downloads,
254            // `or_else` avoids eager clone of `max_version` when `newest_version` is Some.
255            version: c
256                .newest_version
257                .or_else(|| c.max_version.clone())
258                .unwrap_or_else(|| "unknown".to_string()),
259            documentation: c.documentation,
260            max_version: c.max_version,
261            max_stable_version: c.max_stable_version,
262            default_version: c.default_version,
263            recent_downloads: c.recent_downloads,
264            exact_match: c.exact_match,
265            yanked: c.yanked,
266            repository: c.repository,
267            homepage: c.homepage,
268        })
269        .collect();
270
271    Ok(SearchCratesData {
272        query: query.to_string(),
273        page,
274        per_page,
275        sort: sort.to_string(),
276        hits,
277        meta: SearchMeta {
278            total: parsed.meta.total,
279            next_page: parsed.meta.next_page,
280            prev_page: parsed.meta.prev_page,
281        },
282        cache_hit,
283    })
284}
285
286/// Execute search-crates against crates.io (production origin).
287///
288/// # Errors
289///
290/// Propagates URL build, HTTP, UTF-8 decode, and JSON parse failures from
291/// [`search_crates_on_origin`].
292pub async fn search_crates(
293    http: &HttpClient,
294    query: &str,
295    per_page: u32,
296    sort: &str,
297    page: u32,
298) -> AppResult<SearchCratesData> {
299    search_crates_on_origin(
300        http,
301        &format!("https://{HOST_CRATES_IO}"),
302        query,
303        per_page,
304        sort,
305        page,
306    )
307    .await
308}
309
310/// Execute search-crates against a configurable origin (offline mocks).
311///
312/// # Errors
313///
314/// Returns [`ErrorKind::Internal`] when the origin URL is invalid.
315/// Propagates HTTP and parse errors from [`search_crates_at`].
316pub async fn search_crates_on_origin(
317    http: &HttpClient,
318    origin: &str,
319    query: &str,
320    per_page: u32,
321    sort: &str,
322    page: u32,
323) -> AppResult<SearchCratesData> {
324    let (per_page, page) = clamp_search_pagination(per_page, page);
325    let url = planned_url_on_host(origin, query, per_page, sort, page)?;
326    search_crates_at(http, &url, query, per_page, sort, page).await
327}
328
329/// Execute search against a pre-built URL (mock hosts in tests).
330///
331/// # Errors
332///
333/// Propagates HTTP transport errors from [`HttpClient::get_json`].
334/// Maps non-success HTTP statuses via [`AppError::from_http_status`].
335/// Returns [`ErrorKind::Parse`] when the body is not UTF-8 or not valid crates.io JSON.
336pub async fn search_crates_at(
337    http: &HttpClient,
338    url: &Url,
339    query: &str,
340    per_page: u32,
341    sort: &str,
342    page: u32,
343) -> AppResult<SearchCratesData> {
344    let resp = http.get_json(url).await?;
345    if resp.status.as_u16() == 404 {
346        return Err(AppError::from_http_status(404, "crates.io search"));
347    }
348    if !resp.status.is_success() {
349        return Err(AppError::from_http_status(
350            resp.status.as_u16(),
351            "crates.io search",
352        ));
353    }
354    if resp.content_type.is_some() && !content_type_looks_json(resp.content_type.as_deref()) {
355        tracing::warn!(
356            content_type = ?resp.content_type,
357            "unexpected Content-Type from crates.io"
358        );
359    }
360
361    let text = decode_utf8(&resp.body)?;
362    parse_search_body(&text, query, page, per_page, sort, resp.cache_hit)
363}
364
365#[cfg(test)]
366mod tests {
367    use super::*;
368
369    #[test]
370    fn planned_url_clamps_and_encodes() {
371        let u = planned_url("serde json", 0, "alphabetical", 0).unwrap();
372        assert!(u.as_str().contains("per_page=1"));
373        assert!(u.as_str().contains("page=1"));
374        assert!(u.as_str().contains("sort=alphabetical"));
375        assert!(u.as_str().contains("q=serde"));
376        let u2 = planned_url("x", 500, "relevance", 3).unwrap();
377        assert!(u2.as_str().contains("per_page=100"));
378        assert!(u2.as_str().contains("page=3"));
379    }
380
381    #[test]
382    fn clamp_search_pagination_page_and_per_page() {
383        assert_eq!(clamp_search_pagination(0, 0), (1, 1));
384        assert_eq!(clamp_search_pagination(10, 0), (10, 1));
385        assert_eq!(clamp_search_pagination(500, 2), (100, 2));
386        assert_eq!(clamp_search_pagination(25, 3), (25, 3));
387    }
388
389    #[test]
390    fn parse_fixture_serde() {
391        let body = include_str!("../tests/fixtures/crates_io/search_serde.json");
392        let data = parse_search_body(body, "serde", 1, 10, "relevance", false).unwrap();
393        assert_eq!(data.hits.len(), 2);
394        assert_eq!(data.hits[0].name, "serde");
395        assert_eq!(data.hits[0].exact_match, Some(true));
396        assert_eq!(data.hits[0].version, "1.0.210");
397        assert_eq!(data.hits[1].description, "No description available");
398        assert_eq!(data.meta.total, 2);
399        assert_eq!(data.meta.next_page.as_deref(), Some("page=2"));
400    }
401
402    #[test]
403    fn parse_fixture_seek_meta() {
404        let body = include_str!("../tests/fixtures/crates_io/search_seek_meta.json");
405        let data = parse_search_body(body, "example", 1, 10, "downloads", false).unwrap();
406        assert_eq!(
407            data.meta.next_page.as_deref(),
408            Some("seek=ABC123&per_page=10")
409        );
410        assert_eq!(data.meta.prev_page.as_deref(), Some("page=1"));
411        assert_eq!(data.hits[0].version, "0.1.0");
412    }
413
414    #[test]
415    fn parse_invalid_json() {
416        let err = parse_search_body("{", "q", 1, 10, "relevance", false).unwrap_err();
417        assert_eq!(err.kind(), ErrorKind::Parse);
418    }
419
420    #[test]
421    fn parse_missing_versions_uses_unknown() {
422        let body = r#"{"crates":[{"name":"x","downloads":1}],"meta":{"total":1}}"#;
423        let data = parse_search_body(body, "x", 1, 10, "new", false).unwrap();
424        assert_eq!(data.hits[0].version, "unknown");
425        assert!(data.meta.next_page.is_none());
426    }
427
428    #[test]
429    fn planned_url_on_host_http_base() {
430        let u = planned_url_on_host("http://127.0.0.1:9", "q", 10, "new", 1).unwrap();
431        assert!(u.as_str().starts_with("http://127.0.0.1:9/api/v1/crates"));
432    }
433}