docs_mcp/sparse_index/
client.rs

1use reqwest_middleware::ClientWithMiddleware;
2
3use crate::cache::DiskCache;
4use crate::error::{DocsError, Result};
5use super::types::{IndexLine, compute_path};
6
7const INDEX_BASE: &str = "https://index.crates.io";
8
9/// Fetch all index lines for a crate from the sparse index.
10pub async fn fetch_index(
11    name: &str,
12    client: &ClientWithMiddleware,
13    cache: &DiskCache,
14) -> Result<Vec<IndexLine>> {
15    let path = compute_path(name);
16    let url = format!("{INDEX_BASE}/{path}");
17
18    let text = cache.get_text(client, &url).await?;
19    parse_ndjson(&text)
20}
21
22/// Parse NDJSON (newline-delimited JSON) into a list of IndexLine entries.
23pub fn parse_ndjson(text: &str) -> Result<Vec<IndexLine>> {
24    text.lines()
25        .filter(|l| !l.trim().is_empty())
26        .map(|l| serde_json::from_str(l).map_err(DocsError::Json))
27        .collect()
28}
29
30#[cfg(test)]
31mod tests {
32    use super::*;
33
34    #[test]
35    fn test_parse_ndjson_basic() {
36        let ndjson = r#"{"name":"serde","vers":"1.0.0","deps":[],"cksum":"abc","features":{},"yanked":false}
37{"name":"serde","vers":"1.0.1","deps":[],"cksum":"def","features":{},"yanked":false}
38"#;
39        let lines = parse_ndjson(ndjson).unwrap();
40        assert_eq!(lines.len(), 2);
41        assert_eq!(lines[0].vers, "1.0.0");
42        assert_eq!(lines[1].vers, "1.0.1");
43    }
44
45    #[test]
46    fn test_parse_ndjson_with_features() {
47        let ndjson = r#"{"name":"tokio","vers":"1.0.0","deps":[],"cksum":"abc","features":{"full":["rt","sync","io"]},"yanked":false}"#;
48        let lines = parse_ndjson(ndjson).unwrap();
49        assert_eq!(lines.len(), 1);
50        assert!(lines[0].features.contains_key("full"));
51    }
52}
docs_mcp/sparse_index/client.rs

docs_mcp/sparse_index/
client.rs
