crate_paths_cli_core/backend/docrs/
mod.rs

1pub mod error;
2
3use super::BackendError;
4use crate::error::CoreError;
5use crate::item::ItemEntry;
6use crate::parser;
7use error::DocsrsBackendError;
8use reqwest::blocking::Client;
9use std::io::Read as _;
10use zstd::Decoder as ZstdDecoder;
11
12fn fetch_crate_all_items_html(
13    crate_name: &str,
14    crate_version: &str,
15    app_user_agent: &str,
16) -> Result<String, DocsrsBackendError> {
17    let base_url = format!("https://docs.rs/{}/{}/", crate_name, crate_version,);
18
19    let client = Client::builder()
20        .user_agent(app_user_agent)
21        .build()
22        .map_err(DocsrsBackendError::ClientBuild)?;
23
24    // Perform a HEAD request to follow redirects and get the final URL
25    let response = client
26        .head(&base_url)
27        .send()
28        .map_err(|e| DocsrsBackendError::HttpGet(base_url.clone(), e))?
29        .error_for_status()
30        .map_err(|e| DocsrsBackendError::HttpStatus(base_url.clone(), e))?;
31
32    let final_url = response.url();
33    let html_url = final_url
34        .join("all.html")
35        .expect("failed to construct all.html URL");
36    let html_url_str = html_url.to_string();
37
38    let response = client
39        .get(html_url)
40        .send()
41        .map_err(|e| DocsrsBackendError::HttpGet(html_url_str.clone(), e))?
42        .error_for_status()
43        .map_err(|e| DocsrsBackendError::HttpStatus(html_url_str.clone(), e))?;
44
45    eprintln!("final URL: {}", response.url());
46    if let Some(ct) = response.headers().get(reqwest::header::CONTENT_TYPE) {
47        eprintln!("Content-Type: {:?}", ct);
48    }
49    if let Some(enc) = response.headers().get(reqwest::header::CONTENT_ENCODING) {
50        eprintln!("Content-Encoding: {:?}", enc);
51    }
52
53    let content_encoding = response
54        .headers()
55        .get(reqwest::header::CONTENT_ENCODING)
56        .and_then(|h| h.to_str().ok())
57        .map(|s| s.to_ascii_lowercase());
58
59    let raw_bytes = response
60        .bytes()
61        .map_err(|e| DocsrsBackendError::HttpGet(html_url_str.clone(), e))?;
62
63    let html_bytes: Vec<u8> = match content_encoding.as_deref() {
64        Some("zstd") => {
65            let mut decoder =
66                ZstdDecoder::new(raw_bytes.as_ref()).map_err(DocsrsBackendError::Decompress)?;
67            let mut buf = Vec::new();
68            decoder
69                .read_to_end(&mut buf)
70                .map_err(DocsrsBackendError::Decompress)?;
71            buf
72        },
73        _ => raw_bytes.to_vec(),
74    };
75
76    std::str::from_utf8(&html_bytes)
77        .map(|s| s.to_string())
78        .map_err(DocsrsBackendError::Utf8)
79}
80
81pub fn process(
82    crate_name: &str,
83    crate_version: &str,
84    app_user_agent: &str,
85) -> Result<Vec<ItemEntry>, CoreError> {
86    let html_content = fetch_crate_all_items_html(crate_name, crate_version, app_user_agent)
87        .map_err(BackendError::from)
88        .map_err(CoreError::from)?;
89    parser::parse_html_to_items(crate_name, &html_content).map_err(CoreError::from)
90}
crate_paths_cli_core/backend/docrs/mod.rs

crate_paths_cli_core/backend/docrs/
mod.rs
