Skip to main content

docsrs_cli/
crates_io.rs

1//! crates.io search-crates operation.
2
3use serde::{Deserialize, Serialize};
4use url::Url;
5
6use crate::config::{HOST_CRATES_IO, MAX_PER_PAGE};
7use crate::error::{AppError, AppResult, ErrorKind};
8use crate::http::{HttpClient, content_type_looks_json, decode_utf8};
9
10/// One hit from crates.io search.
11#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
12pub struct CrateSearchHit {
13    /// Crate name on crates.io.
14    pub name: String,
15    /// Short crate description (may be empty).
16    pub description: String,
17    /// Total download count.
18    pub downloads: u64,
19    /// Newest or default version string for display.
20    pub version: String,
21    /// Documentation URL when provided by crates.io.
22    #[serde(skip_serializing_if = "Option::is_none")]
23    pub documentation: Option<String>,
24    /// Highest published version when present.
25    #[serde(skip_serializing_if = "Option::is_none")]
26    pub max_version: Option<String>,
27    /// Highest stable version when present.
28    #[serde(skip_serializing_if = "Option::is_none")]
29    pub max_stable_version: Option<String>,
30    /// Default version selected by crates.io.
31    #[serde(skip_serializing_if = "Option::is_none")]
32    pub default_version: Option<String>,
33    /// Recent download count when present.
34    #[serde(skip_serializing_if = "Option::is_none")]
35    pub recent_downloads: Option<u64>,
36    /// Whether the query matched the crate name exactly.
37    #[serde(skip_serializing_if = "Option::is_none")]
38    pub exact_match: Option<bool>,
39    /// Whether the default version is yanked.
40    #[serde(skip_serializing_if = "Option::is_none")]
41    pub yanked: Option<bool>,
42    /// Source repository URL when present.
43    #[serde(skip_serializing_if = "Option::is_none")]
44    pub repository: Option<String>,
45    /// Project homepage URL when present.
46    #[serde(skip_serializing_if = "Option::is_none")]
47    pub homepage: Option<String>,
48}
49
50/// Pagination meta from crates.io (may contain `page=` or `seek=` tokens).
51#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
52pub struct SearchMeta {
53    /// Total hits reported by crates.io.
54    pub total: u64,
55    /// Opaque next-page token when more results exist.
56    #[serde(skip_serializing_if = "Option::is_none")]
57    pub next_page: Option<String>,
58    /// Opaque previous-page token when available.
59    #[serde(skip_serializing_if = "Option::is_none")]
60    pub prev_page: Option<String>,
61}
62
63/// Full search-crates data payload.
64#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
65pub struct SearchCratesData {
66    /// Original search query string.
67    pub query: String,
68    /// 1-based page number used for the request.
69    pub page: u32,
70    /// Page size used for the request.
71    pub per_page: u32,
72    /// Sort order API token.
73    pub sort: String,
74    /// Search hits on this page.
75    pub hits: Vec<CrateSearchHit>,
76    /// Pagination metadata from crates.io.
77    pub meta: SearchMeta,
78    /// True when the HTTP body was served from the local disk cache.
79    pub cache_hit: bool,
80}
81
82/// Echo fields for `search-crates` taken from the **effective request URL**
83/// (single source of truth after URL planning, including `--page-token`).
84#[derive(Debug, Clone, PartialEq, Eq)]
85pub struct SearchEcho {
86    /// Query string (`q=`).
87    pub query: String,
88    /// 1-based page.
89    pub page: u32,
90    /// Page size.
91    pub per_page: u32,
92    /// Sort token.
93    pub sort: String,
94}
95
96impl SearchEcho {
97    /// Build from CLI-side fallbacks before URL planning.
98    pub fn from_cli(query: &str, page: u32, per_page: u32, sort: &str) -> Self {
99        let (per_page, page) = clamp_search_pagination(per_page, page);
100        Self {
101            query: query.to_string(),
102            page,
103            per_page,
104            sort: sort.to_string(),
105        }
106    }
107}
108
109/// Derive echo params from the planned URL query string.
110///
111/// Missing pairs fall back to `fallback` so pure `seek=` tokens and partial
112/// tokens still produce a coherent agent-facing payload.
113pub fn echo_params_from_url(url: &Url, fallback: &SearchEcho) -> SearchEcho {
114    let mut query = fallback.query.clone();
115    let mut page = fallback.page;
116    let mut per_page = fallback.per_page;
117    let mut sort = fallback.sort.clone();
118    for (k, v) in url.query_pairs() {
119        match k.as_ref() {
120            "q" => query = v.into_owned(),
121            "page" => {
122                if let Ok(n) = v.parse::<u32>() {
123                    page = n.max(1);
124                }
125            }
126            "per_page" => {
127                if let Ok(n) = v.parse::<u32>() {
128                    per_page = n.clamp(1, MAX_PER_PAGE);
129                }
130            }
131            "sort" => sort = v.into_owned(),
132            _ => {}
133        }
134    }
135    let (per_page, page) = clamp_search_pagination(per_page, page);
136    SearchEcho {
137        query,
138        page,
139        per_page,
140        sort,
141    }
142}
143
144#[derive(Debug, Deserialize)]
145struct ApiResponse {
146    crates: Vec<ApiCrate>,
147    meta: ApiMeta,
148}
149
150#[derive(Debug, Deserialize)]
151struct ApiCrate {
152    name: String,
153    description: Option<String>,
154    downloads: u64,
155    newest_version: Option<String>,
156    max_version: Option<String>,
157    max_stable_version: Option<String>,
158    default_version: Option<String>,
159    documentation: Option<String>,
160    recent_downloads: Option<u64>,
161    exact_match: Option<bool>,
162    yanked: Option<bool>,
163    repository: Option<String>,
164    homepage: Option<String>,
165}
166
167#[derive(Debug, Deserialize)]
168struct ApiMeta {
169    total: u64,
170    next_page: Option<String>,
171    prev_page: Option<String>,
172}
173
174/// Clamp crates.io pagination to product bounds (`per_page` 1..=100, `page` >= 1).
175///
176/// Dry-run `planned_params` and the request URL MUST use these same values so
177/// agents never see `page=0` in params while the URL has `page=1`.
178pub fn clamp_search_pagination(per_page: u32, page: u32) -> (u32, u32) {
179    (per_page.clamp(1, MAX_PER_PAGE), page.max(1))
180}
181
182/// Build the planned crates.io search URL.
183///
184/// # Errors
185///
186/// Returns [`ErrorKind::Internal`] when the base URL cannot be parsed.
187pub fn planned_url(query: &str, per_page: u32, sort: &str, page: u32) -> AppResult<Url> {
188    planned_url_on_host(HOST_CRATES_IO, query, per_page, sort, page)
189}
190
191/// Build search URL against an arbitrary host (used by tests / local mocks).
192///
193/// # Errors
194///
195/// Returns [`ErrorKind::Internal`] when `host` does not form a valid base URL.
196pub fn planned_url_on_host(
197    host: &str,
198    query: &str,
199    per_page: u32,
200    sort: &str,
201    page: u32,
202) -> AppResult<Url> {
203    let (per_page, page) = clamp_search_pagination(per_page, page);
204    let scheme = if host.starts_with("http://") || host.starts_with("https://") {
205        ""
206    } else {
207        "https://"
208    };
209    let base = if scheme.is_empty() {
210        format!("{host}/api/v1/crates")
211    } else {
212        format!("{scheme}{host}/api/v1/crates")
213    };
214    let mut url = Url::parse(&base)
215        .map_err(|e| AppError::with_source(ErrorKind::Internal, "invalid crates.io base URL", e))?;
216    {
217        let mut q = url.query_pairs_mut();
218        q.append_pair("q", query);
219        q.append_pair("per_page", &per_page.to_string());
220        q.append_pair("sort", sort);
221        q.append_pair("page", &page.to_string());
222    }
223    Ok(url)
224}
225
226/// Build search URL using an opaque `next_page` / `prev_page` token from crates.io meta.
227///
228/// Accepts:
229/// - full query string (`?q=…&page=2` or `q=…&page=2`)
230/// - pure page number string
231/// - `seek=…` tokens
232///
233/// # Errors
234///
235/// Returns [`ErrorKind::Internal`] when the base URL is invalid.
236/// Returns [`ErrorKind::InvalidInput`] when the token cannot form a valid URL.
237pub fn planned_url_with_page_token(
238    host: &str,
239    query: &str,
240    per_page: u32,
241    sort: &str,
242    page_token: &str,
243) -> AppResult<Url> {
244    let token = page_token.trim();
245    if token.is_empty() {
246        return Err(AppError::new(
247            ErrorKind::InvalidInput,
248            "page token is empty",
249        ));
250    }
251    // Pure numeric → page number
252    if let Ok(page) = token.parse::<u32>() {
253        return planned_url_on_host(host, query, per_page, sort, page);
254    }
255    let scheme = if host.starts_with("http://") || host.starts_with("https://") {
256        ""
257    } else {
258        "https://"
259    };
260    let base = if scheme.is_empty() {
261        format!("{host}/api/v1/crates")
262    } else {
263        format!("{scheme}{host}/api/v1/crates")
264    };
265    let base_url = Url::parse(&base)
266        .map_err(|e| AppError::with_source(ErrorKind::Internal, "invalid crates.io base URL", e))?;
267
268    let qs = token.trim_start_matches('?');
269    // If token already carries a full query, use it as the request query.
270    if qs.contains('=') {
271        let mut url = base_url;
272        url.set_query(Some(qs));
273        return Ok(url);
274    }
275    // Opaque seek-like token
276    let (per_page, _) = clamp_search_pagination(per_page, 1);
277    let mut url = base_url;
278    {
279        let mut q = url.query_pairs_mut();
280        q.append_pair("q", query);
281        q.append_pair("per_page", &per_page.to_string());
282        q.append_pair("sort", sort);
283        q.append_pair("seek", qs);
284    }
285    Ok(url)
286}
287/// Parse crates.io JSON body into product types (offline-testable).
288///
289/// # Errors
290///
291/// Returns [`ErrorKind::Parse`] when `text` is not valid crates.io search JSON.
292pub fn parse_search_body(
293    text: &str,
294    query: &str,
295    page: u32,
296    per_page: u32,
297    sort: &str,
298    cache_hit: bool,
299) -> AppResult<SearchCratesData> {
300    let (per_page, page) = clamp_search_pagination(per_page, page);
301    let parsed: ApiResponse = serde_json::from_str(text).map_err(|e| {
302        AppError::with_source(ErrorKind::Parse, "failed to parse crates.io JSON", e)
303    })?;
304
305    let hits = parsed
306        .crates
307        .into_iter()
308        .map(|c| CrateSearchHit {
309            name: c.name,
310            description: c
311                .description
312                .filter(|s| !s.is_empty())
313                .unwrap_or_else(|| "No description available".to_string()),
314            downloads: c.downloads,
315            // `or_else` avoids eager clone of `max_version` when `newest_version` is Some.
316            version: c
317                .newest_version
318                .or_else(|| c.max_version.clone())
319                .unwrap_or_else(|| "unknown".to_string()),
320            documentation: c.documentation,
321            max_version: c.max_version,
322            max_stable_version: c.max_stable_version,
323            default_version: c.default_version,
324            recent_downloads: c.recent_downloads,
325            exact_match: c.exact_match,
326            yanked: c.yanked,
327            repository: c.repository,
328            homepage: c.homepage,
329        })
330        .collect();
331
332    Ok(SearchCratesData {
333        query: query.to_string(),
334        page,
335        per_page,
336        sort: sort.to_string(),
337        hits,
338        meta: SearchMeta {
339            total: parsed.meta.total,
340            next_page: parsed.meta.next_page,
341            prev_page: parsed.meta.prev_page,
342        },
343        cache_hit,
344    })
345}
346
347/// Execute search-crates against crates.io (production origin).
348///
349/// # Errors
350///
351/// Propagates URL build, HTTP, UTF-8 decode, and JSON parse failures from
352/// [`search_crates_on_origin`].
353pub async fn search_crates(
354    http: &HttpClient,
355    query: &str,
356    per_page: u32,
357    sort: &str,
358    page: u32,
359) -> AppResult<SearchCratesData> {
360    search_crates_on_origin(
361        http,
362        &format!("https://{HOST_CRATES_IO}"),
363        query,
364        per_page,
365        sort,
366        page,
367    )
368    .await
369}
370
371/// Execute search-crates against a configurable origin (offline mocks).
372///
373/// # Errors
374///
375/// Returns [`ErrorKind::Internal`] when the origin URL is invalid.
376/// Propagates HTTP and parse errors from [`search_crates_at`].
377pub async fn search_crates_on_origin(
378    http: &HttpClient,
379    origin: &str,
380    query: &str,
381    per_page: u32,
382    sort: &str,
383    page: u32,
384) -> AppResult<SearchCratesData> {
385    let (per_page, page) = clamp_search_pagination(per_page, page);
386    let url = planned_url_on_host(origin, query, per_page, sort, page)?;
387    search_crates_at(http, &url, query, per_page, sort, page).await
388}
389
390/// Execute search against a pre-built URL (mock hosts in tests).
391///
392/// Echo fields (`query`/`page`/`per_page`/`sort`) are taken from the **URL**
393/// via [`echo_params_from_url`], with CLI values as fallback for missing pairs.
394///
395/// # Errors
396///
397/// Propagates HTTP transport errors from [`HttpClient::get_json`].
398/// Maps non-success HTTP statuses via [`AppError::from_http_status`].
399/// Returns [`ErrorKind::Parse`] when the body is not UTF-8 or not valid crates.io JSON.
400pub async fn search_crates_at(
401    http: &HttpClient,
402    url: &Url,
403    query: &str,
404    per_page: u32,
405    sort: &str,
406    page: u32,
407) -> AppResult<SearchCratesData> {
408    let fallback = SearchEcho::from_cli(query, page, per_page, sort);
409    let echo = echo_params_from_url(url, &fallback);
410    let resp = http.get_json(url).await?;
411    if resp.status.as_u16() == 404 {
412        return Err(AppError::from_http_status(404, "crates.io search"));
413    }
414    if !resp.status.is_success() {
415        return Err(AppError::from_http_status(
416            resp.status.as_u16(),
417            "crates.io search",
418        ));
419    }
420    if resp.content_type.is_some() && !content_type_looks_json(resp.content_type.as_deref()) {
421        tracing::warn!(
422            content_type = ?resp.content_type,
423            "unexpected Content-Type from crates.io"
424        );
425    }
426
427    let text = decode_utf8(&resp.body)?;
428    parse_search_body(
429        &text,
430        &echo.query,
431        echo.page,
432        echo.per_page,
433        &echo.sort,
434        resp.cache_hit,
435    )
436}
437
438#[cfg(test)]
439mod tests {
440    use super::*;
441
442    #[test]
443    fn planned_url_clamps_and_encodes() {
444        let u = planned_url("serde json", 0, "alphabetical", 0).unwrap();
445        assert!(u.as_str().contains("per_page=1"));
446        assert!(u.as_str().contains("page=1"));
447        assert!(u.as_str().contains("sort=alphabetical"));
448        assert!(u.as_str().contains("q=serde"));
449        let u2 = planned_url("x", 500, "relevance", 3).unwrap();
450        assert!(u2.as_str().contains("per_page=100"));
451        assert!(u2.as_str().contains("page=3"));
452    }
453
454    #[test]
455    fn clamp_search_pagination_page_and_per_page() {
456        assert_eq!(clamp_search_pagination(0, 0), (1, 1));
457        assert_eq!(clamp_search_pagination(10, 0), (10, 1));
458        assert_eq!(clamp_search_pagination(500, 2), (100, 2));
459        assert_eq!(clamp_search_pagination(25, 3), (25, 3));
460    }
461
462    #[test]
463    fn parse_fixture_serde() {
464        let body = include_str!("../tests/fixtures/crates_io/search_serde.json");
465        let data = parse_search_body(body, "serde", 1, 10, "relevance", false).unwrap();
466        assert_eq!(data.hits.len(), 2);
467        assert_eq!(data.hits[0].name, "serde");
468        assert_eq!(data.hits[0].exact_match, Some(true));
469        assert_eq!(data.hits[0].version, "1.0.210");
470        assert_eq!(data.hits[1].description, "No description available");
471        assert_eq!(data.meta.total, 2);
472        assert_eq!(data.meta.next_page.as_deref(), Some("page=2"));
473    }
474
475    #[test]
476    fn parse_fixture_seek_meta() {
477        let body = include_str!("../tests/fixtures/crates_io/search_seek_meta.json");
478        let data = parse_search_body(body, "example", 1, 10, "downloads", false).unwrap();
479        assert_eq!(
480            data.meta.next_page.as_deref(),
481            Some("seek=ABC123&per_page=10")
482        );
483        assert_eq!(data.meta.prev_page.as_deref(), Some("page=1"));
484        assert_eq!(data.hits[0].version, "0.1.0");
485    }
486
487    #[test]
488    fn parse_invalid_json() {
489        let err = parse_search_body("{", "q", 1, 10, "relevance", false).unwrap_err();
490        assert_eq!(err.kind(), ErrorKind::Parse);
491    }
492
493    #[test]
494    fn parse_missing_versions_uses_unknown() {
495        let body = r#"{"crates":[{"name":"x","downloads":1}],"meta":{"total":1}}"#;
496        let data = parse_search_body(body, "x", 1, 10, "new", false).unwrap();
497        assert_eq!(data.hits[0].version, "unknown");
498        assert!(data.meta.next_page.is_none());
499    }
500
501    #[test]
502    fn planned_url_on_host_http_base() {
503        let u = planned_url_on_host("http://127.0.0.1:9", "q", 10, "new", 1).unwrap();
504        assert!(u.as_str().starts_with("http://127.0.0.1:9/api/v1/crates"));
505    }
506
507    #[test]
508    fn echo_params_from_page_token_url() {
509        let fallback = SearchEcho::from_cli("", 1, 10, "relevance");
510        let url = planned_url_with_page_token(
511            "https://crates.io",
512            "",
513            10,
514            "relevance",
515            "?q=serde&per_page=2&sort=relevance&page=2",
516        )
517        .unwrap();
518        let echo = echo_params_from_url(&url, &fallback);
519        assert_eq!(echo.query, "serde");
520        assert_eq!(echo.page, 2);
521        assert_eq!(echo.per_page, 2);
522        assert_eq!(echo.sort, "relevance");
523    }
524
525    #[test]
526    fn echo_params_fallback_when_pairs_missing() {
527        let fallback = SearchEcho::from_cli("local", 3, 25, "downloads");
528        let url = Url::parse("https://crates.io/api/v1/crates?seek=ABC").unwrap();
529        let echo = echo_params_from_url(&url, &fallback);
530        assert_eq!(echo.query, "local");
531        assert_eq!(echo.page, 3);
532        assert_eq!(echo.per_page, 25);
533        assert_eq!(echo.sort, "downloads");
534    }
535}