upstream_ontologist/providers/
rust.rs

1use crate::{
2    Certainty, GuesserSettings, Person, ProviderError, UpstreamDatum, UpstreamDatumWithMetadata,
3    UpstreamMetadata,
4};
5use serde::Deserialize;
6use std::collections::HashMap;
7
8/// Extracts upstream metadata from Cargo.toml file
9#[cfg(feature = "cargo")]
10pub fn guess_from_cargo(
11    path: &std::path::Path,
12    _settings: &GuesserSettings,
13) -> std::result::Result<Vec<UpstreamDatumWithMetadata>, ProviderError> {
14    // see https://doc.rust-lang.org/cargo/reference/manifest.html
15    let mut manifest = cargo_toml::Manifest::from_path(path)
16        .map_err(|e| ProviderError::ParseError(e.to_string()))?;
17
18    // Try to resolve workspace inheritance from a parent workspace
19    if manifest.needs_workspace_inheritance() {
20        if let Some(workspace_path) = path.parent().and_then(|p| p.parent()) {
21            let workspace_toml = workspace_path.join("Cargo.toml");
22            if workspace_toml.exists() {
23                if let Ok(workspace_manifest) = cargo_toml::Manifest::from_path(&workspace_toml) {
24                    let _ = manifest.complete_from_path_and_workspace(
25                        path,
26                        Some((&workspace_manifest, workspace_path)),
27                    );
28                }
29            }
30        }
31    }
32
33    let package = match manifest.package {
34        Some(ref p) => p,
35        None => {
36            log::debug!("No package section in Cargo.toml");
37            return Ok(Vec::new());
38        }
39    };
40
41    let mut results = Vec::new();
42
43    let name = &package.name;
44    results.push(UpstreamDatumWithMetadata {
45        datum: UpstreamDatum::Name(name.clone()),
46        certainty: Some(Certainty::Certain),
47        origin: Some(path.into()),
48    });
49    results.push(UpstreamDatumWithMetadata {
50        datum: UpstreamDatum::CargoCrate(name.clone()),
51        certainty: Some(Certainty::Certain),
52        origin: Some(path.into()),
53    });
54
55    if let Some(description) = package.description() {
56        results.push(UpstreamDatumWithMetadata {
57            datum: UpstreamDatum::Summary(description.to_string()),
58            certainty: Some(Certainty::Certain),
59            origin: Some(path.into()),
60        });
61    }
62
63    if let Some(homepage) = package.homepage() {
64        results.push(UpstreamDatumWithMetadata {
65            datum: UpstreamDatum::Homepage(homepage.to_string()),
66            certainty: Some(Certainty::Certain),
67            origin: Some(path.into()),
68        });
69    }
70
71    if let Some(license) = package.license() {
72        results.push(UpstreamDatumWithMetadata {
73            datum: UpstreamDatum::License(license.to_string()),
74            certainty: Some(Certainty::Certain),
75            origin: Some(path.into()),
76        });
77    }
78
79    if let Some(repository) = package.repository() {
80        results.push(UpstreamDatumWithMetadata {
81            datum: UpstreamDatum::Repository(repository.to_string()),
82            certainty: Some(Certainty::Certain),
83            origin: Some(path.into()),
84        });
85    }
86
87    let version = package.version();
88    results.push(UpstreamDatumWithMetadata {
89        datum: UpstreamDatum::Version(version.to_string()),
90        certainty: Some(Certainty::Certain),
91        origin: Some(path.into()),
92    });
93
94    let authors = package.authors();
95    if !authors.is_empty() {
96        let authors = authors.iter().map(|a| Person::from(a.as_str())).collect();
97        results.push(UpstreamDatumWithMetadata {
98            datum: UpstreamDatum::Author(authors),
99            certainty: Some(Certainty::Certain),
100            origin: Some(path.into()),
101        });
102    }
103
104    Ok(results)
105}
106
107/// Translates crate names with dashes to their canonical form on crates.io
108pub async fn cargo_translate_dashes(
109    crate_name: &str,
110) -> Result<Option<String>, crate::HTTPJSONError> {
111    let url: url::Url = format!("https://crates.io/api/v1/crates?q={}", crate_name)
112        .parse()
113        .expect("static crates.io URL template should always be valid");
114    let json: serde_json::Value = crate::load_json_url(&url, None).await?;
115
116    // Navigate through the JSON response to find the crate name.
117    if let Some(crates) = json.get("crates").and_then(|c| c.as_array()) {
118        for krate in crates {
119            if let Some(name) = krate.get("id").and_then(|n| n.as_str()) {
120                return Ok(Some(name.to_string()));
121            }
122        }
123    }
124
125    Ok(None)
126}
127
128/// Crate metadata from crates.io
129#[derive(Deserialize)]
130pub struct Crate {
131    /// Crate badges
132    pub badges: Vec<String>,
133    /// Creation timestamp
134    pub created_at: String,
135    /// Crate description
136    pub description: Option<String>,
137    /// Documentation URL
138    pub documentation: Option<String>,
139    /// Total downloads
140    pub downloads: i64,
141    /// Homepage URL
142    pub homepage: Option<String>,
143    /// Crate identifier
144    pub id: String,
145    /// Keywords
146    pub keywords: Vec<String>,
147    /// License identifier
148    pub license: Option<String>,
149    /// Various links
150    pub links: HashMap<String, Option<String>>,
151    /// Maximum stable version
152    pub max_stable_version: semver::Version,
153    /// Maximum version
154    pub max_version: semver::Version,
155    /// Crate name
156    pub name: String,
157    /// Newest version
158    pub newest_version: semver::Version,
159    /// Recent downloads
160    pub recent_downloads: i64,
161    /// Repository URL
162    pub repository: Option<String>,
163    /// Last update timestamp
164    pub updated_at: String,
165    /// Version IDs
166    pub versions: Option<Vec<i32>>,
167}
168
169/// User information from crates.io
170#[derive(Deserialize)]
171pub struct User {
172    /// User avatar URL
173    pub avatar: String,
174    /// User ID
175    pub id: i32,
176    /// User login name
177    pub login: String,
178    /// User display name
179    pub name: String,
180    /// User profile URL
181    pub url: String,
182}
183
184/// Audit action information
185#[derive(Deserialize)]
186pub struct AuditAction {
187    /// Action type
188    pub action: String,
189    /// Timestamp of the action
190    pub time: String,
191    /// User who performed the action
192    pub user: User,
193}
194
195/// Information about a specific version of a crate
196#[derive(Deserialize)]
197pub struct CrateVersion {
198    /// Audit actions for this version
199    pub audit_actions: Vec<AuditAction>,
200    /// Names of binary targets
201    pub bin_names: Vec<String>,
202    /// Checksum of the crate
203    pub checksum: String,
204    /// Name of the crate
205    #[serde(rename = "crate")]
206    pub crate_: String,
207    /// Size of the crate in bytes
208    pub crate_size: i64,
209    /// Creation timestamp
210    pub created_at: String,
211    /// Download path
212    pub dl_path: String,
213    /// Number of downloads
214    pub downloads: i64,
215    /// Feature flags
216    pub features: HashMap<String, Vec<String>>,
217    /// Whether the crate has a library
218    pub has_lib: bool,
219    /// Version ID
220    pub id: i32,
221    /// Library links
222    pub lib_links: Option<HashMap<String, String>>,
223    /// License identifier
224    pub license: Option<String>,
225    /// Various links
226    pub links: HashMap<String, Option<String>>,
227    /// Version number
228    pub num: semver::Version,
229    /// User who published this version
230    pub published_by: Option<User>,
231    /// Path to README file
232    pub readme_path: String,
233    /// Minimum Rust version required
234    pub rust_version: Option<String>,
235    /// Last update timestamp
236    pub updated_at: String,
237    /// Whether this version is yanked
238    pub yanked: bool,
239}
240
241/// Category information from crates.io
242#[derive(Deserialize)]
243pub struct Category {
244    /// Category identifier
245    pub id: String,
246    /// Category display name
247    pub category: String,
248}
249
250/// Keyword information from crates.io
251#[derive(Deserialize)]
252pub struct Keyword {
253    /// Keyword identifier
254    pub id: String,
255    /// Keyword text
256    pub keyword: String,
257}
258
259/// Information about a crate from crates.io
260#[derive(Deserialize)]
261pub struct CrateInfo {
262    /// Categories the crate belongs to
263    pub categories: Vec<Category>,
264    #[serde(rename = "crate")]
265    crate_: Crate,
266    /// Keywords associated with the crate
267    pub keywords: Vec<Keyword>,
268    /// All versions of the crate
269    pub versions: Vec<CrateVersion>,
270}
271
272impl TryFrom<CrateInfo> for UpstreamMetadata {
273    type Error = crate::ProviderError;
274
275    fn try_from(value: CrateInfo) -> Result<Self, Self::Error> {
276        let mut ret = UpstreamMetadata::default();
277
278        ret.insert(UpstreamDatumWithMetadata {
279            datum: UpstreamDatum::Name(value.crate_.name.to_string()),
280            certainty: Some(Certainty::Certain),
281            origin: None,
282        });
283
284        if let Some(homepage) = value.crate_.homepage {
285            ret.insert(UpstreamDatumWithMetadata {
286                datum: UpstreamDatum::Homepage(homepage),
287                certainty: Some(Certainty::Certain),
288                origin: None,
289            });
290        }
291
292        if let Some(repository) = value.crate_.repository {
293            ret.insert(UpstreamDatumWithMetadata {
294                datum: UpstreamDatum::Repository(repository),
295                certainty: Some(Certainty::Certain),
296                origin: None,
297            });
298        }
299
300        if let Some(description) = value.crate_.description {
301            ret.insert(UpstreamDatumWithMetadata {
302                datum: UpstreamDatum::Summary(description),
303                certainty: Some(Certainty::Certain),
304                origin: None,
305            });
306        }
307
308        if let Some(license) = value.crate_.license {
309            ret.insert(UpstreamDatumWithMetadata {
310                datum: UpstreamDatum::License(license),
311                certainty: Some(Certainty::Certain),
312                origin: None,
313            });
314        }
315
316        ret.insert(UpstreamDatumWithMetadata {
317            datum: UpstreamDatum::Version(value.crate_.newest_version.to_string()),
318            certainty: Some(Certainty::Certain),
319            origin: None,
320        });
321
322        Ok(ret)
323    }
324}
325
326/// Loads crate information from crates.io API
327pub async fn load_crate_info(cratename: &str) -> Result<Option<CrateInfo>, crate::ProviderError> {
328    let http_url: url::Url = format!("https://crates.io/api/v1/crates/{}", cratename)
329        .parse()
330        .map_err(|e: url::ParseError| crate::ProviderError::Other(e.to_string()))?;
331
332    let data = crate::load_json_url(&http_url, None).await?;
333
334    serde_json::from_value(data)
335        .map(Some)
336        .map_err(|e| crate::ProviderError::ParseError(format!("Failed to parse crate data: {}", e)))
337}
338
339/// A dependency of a crate version, as recorded in the registry index.
340///
341/// See <https://doc.rust-lang.org/cargo/reference/registry-index.html>.
342#[derive(Deserialize, Debug, Clone)]
343pub struct IndexDependency {
344    /// Name of the dependency (the renamed name if it was renamed).
345    pub name: String,
346    /// SemVer requirement for this dependency.
347    pub req: String,
348    /// Features enabled for this dependency.
349    #[serde(default)]
350    pub features: Vec<String>,
351    /// Whether this is an optional dependency.
352    #[serde(default)]
353    pub optional: bool,
354    /// Whether the dependency's default features are enabled.
355    #[serde(default = "default_true")]
356    pub default_features: bool,
357    /// Target platform expression this dependency applies to, if any.
358    #[serde(default)]
359    pub target: Option<String>,
360    /// Dependency kind: `normal`, `dev` or `build`.
361    #[serde(default)]
362    pub kind: Option<String>,
363    /// Registry index URL, or `None` for the current registry.
364    #[serde(default)]
365    pub registry: Option<String>,
366    /// Original package name if the dependency was renamed.
367    #[serde(default)]
368    pub package: Option<String>,
369}
370
371fn default_true() -> bool {
372    true
373}
374
375/// A single crate version entry from the crates.io sparse index.
376///
377/// The sparse index only carries the data needed for dependency resolution
378/// (name, version, dependencies, features, checksum). It does not include the
379/// rich metadata served by the crates.io REST API such as the homepage,
380/// repository, description or license; use [`load_crate_info`] for those.
381#[derive(Deserialize, Debug, Clone)]
382pub struct IndexEntry {
383    /// Name of the crate.
384    pub name: String,
385    /// Version of this entry.
386    pub vers: semver::Version,
387    /// Dependencies of this version.
388    pub deps: Vec<IndexDependency>,
389    /// SHA256 checksum of the `.crate` file.
390    pub cksum: String,
391    /// Map of feature names to the features or dependencies they enable.
392    #[serde(default)]
393    pub features: HashMap<String, Vec<String>>,
394    /// Map of features using the 1.60+ extended syntax, if present.
395    #[serde(default)]
396    pub features2: Option<HashMap<String, Vec<String>>>,
397    /// Whether this version has been yanked.
398    #[serde(default)]
399    pub yanked: bool,
400    /// The `links` value from the manifest, if any.
401    #[serde(default)]
402    pub links: Option<String>,
403    /// Schema version of this entry, defaulting to 1.
404    #[serde(default)]
405    pub v: Option<u32>,
406    /// Minimum supported Rust version, if declared.
407    #[serde(default)]
408    pub rust_version: Option<String>,
409}
410
411/// Computes the registry index path for a crate name, per RFC 2789.
412///
413/// The same layout is used by the sparse index and the `crates.io-index` git
414/// mirror. Names are lower-cased and bucketed by length: 1-3 character names
415/// use dedicated prefixes, longer names are split into two-character
416/// directories.
417fn index_path(name: &str) -> String {
418    let lower = name.to_lowercase();
419    match lower.len() {
420        1 => format!("1/{}", lower),
421        2 => format!("2/{}", lower),
422        3 => format!("3/{}/{}", &lower[..1], lower),
423        _ => format!("{}/{}/{}", &lower[..2], &lower[2..4], lower),
424    }
425}
426
427/// Loads all version entries for a crate from the crates.io registry index.
428///
429/// The index data is fetched from the `rust-lang/crates.io-index` repository on
430/// GitHub, which mirrors the same per-crate metadata as the sparse index at
431/// [`index.crates.io`](https://index.crates.io/).
432///
433/// Returns `None` if the crate is not present in the index. The entries are
434/// returned in the order they appear in the index, which is publication order
435/// (oldest first).
436pub async fn load_index_info(
437    cratename: &str,
438) -> Result<Option<Vec<IndexEntry>>, crate::ProviderError> {
439    let body = match crate::github::download_raw_file(
440        "rust-lang",
441        "crates.io-index",
442        "master",
443        &index_path(cratename),
444    )
445    .await
446    {
447        Ok(body) => body,
448        Err(crate::HTTPJSONError::Error { status: 404, .. }) => return Ok(None),
449        Err(e) => return Err(e.into()),
450    };
451
452    let mut entries = Vec::new();
453    for line in body.lines() {
454        if line.trim().is_empty() {
455            continue;
456        }
457        let entry: IndexEntry = serde_json::from_str(line).map_err(|e| {
458            crate::ProviderError::ParseError(format!("Failed to parse index entry: {}", e))
459        })?;
460        entries.push(entry);
461    }
462
463    Ok(Some(entries))
464}
465
466// TODO: dedupe with TryFrom implementation above
467fn parse_crates_io(data: &CrateInfo) -> Vec<UpstreamDatum> {
468    let crate_data = &data.crate_;
469    let mut results = Vec::new();
470    results.push(UpstreamDatum::Name(crate_data.name.to_string()));
471    if let Some(homepage) = crate_data.homepage.as_ref() {
472        results.push(UpstreamDatum::Homepage(homepage.to_string()));
473    }
474    if let Some(repository) = crate_data.repository.as_ref() {
475        results.push(UpstreamDatum::Repository(repository.to_string()));
476    }
477    if let Some(description) = crate_data.description.as_ref() {
478        results.push(UpstreamDatum::Summary(description.to_string()));
479    }
480    if let Some(license) = crate_data.license.as_ref() {
481        results.push(UpstreamDatum::License(license.to_string()));
482    }
483    results.push(UpstreamDatum::Version(
484        crate_data.newest_version.to_string(),
485    ));
486
487    results
488}
489
490/// Crates.io metadata provider
491pub struct CratesIo;
492
493impl Default for CratesIo {
494    fn default() -> Self {
495        Self::new()
496    }
497}
498
499impl CratesIo {
500    /// Creates a new CratesIo provider
501    pub fn new() -> Self {
502        Self
503    }
504}
505
506#[async_trait::async_trait]
507impl crate::ThirdPartyRepository for CratesIo {
508    fn name(&self) -> &'static str {
509        "crates.io"
510    }
511
512    fn max_supported_certainty(&self) -> Certainty {
513        Certainty::Certain
514    }
515
516    fn supported_fields(&self) -> &'static [&'static str] {
517        &["Homepage", "Name", "Repository", "Version", "Summary"][..]
518    }
519
520    async fn guess_metadata(&self, name: &str) -> Result<Vec<UpstreamDatum>, ProviderError> {
521        let data = load_crate_info(name).await?;
522        if data.is_none() {
523            return Ok(Vec::new());
524        }
525        Ok(parse_crates_io(&data.unwrap()))
526    }
527}
528
529/// Fetches upstream metadata for a crate from crates.io
530pub async fn remote_crate_data(name: &str) -> Result<UpstreamMetadata, crate::ProviderError> {
531    let data = load_crate_info(name).await?;
532
533    if let Some(data) = data {
534        Ok(data.try_into()?)
535    } else {
536        Ok(UpstreamMetadata::default())
537    }
538}
539
540#[cfg(test)]
541mod crates_io_tests {
542    use super::*;
543
544    #[test]
545    fn test_load_crate_info() {
546        let data = include_str!("../testdata/crates.io.json");
547
548        let crate_info: CrateInfo = serde_json::from_str(data).unwrap();
549
550        assert_eq!(crate_info.crate_.name, "breezy");
551    }
552
553    #[test]
554    fn test_index_path() {
555        assert_eq!(index_path("a"), "1/a");
556        assert_eq!(index_path("ab"), "2/ab");
557        assert_eq!(index_path("abc"), "3/a/abc");
558        assert_eq!(index_path("serde"), "se/rd/serde");
559        assert_eq!(index_path("Inflector"), "in/fl/inflector");
560        assert_eq!(index_path("cargo-edit"), "ca/rg/cargo-edit");
561    }
562
563    #[test]
564    fn test_parse_index_entry() {
565        let line = r#"{"name":"serde","vers":"1.0.0","deps":[{"name":"serde_derive","req":"^1.0","features":[],"optional":true,"default_features":true,"target":null,"kind":"normal"}],"cksum":"00000000000000000000000000000000000000000000000000000000000000aa","features":{"derive":["serde_derive"],"default":["std"]},"yanked":false}"#;
566
567        let entry: IndexEntry = serde_json::from_str(line).unwrap();
568
569        assert_eq!(entry.name, "serde");
570        assert_eq!(entry.vers, semver::Version::new(1, 0, 0));
571        assert_eq!(entry.deps.len(), 1);
572        assert_eq!(entry.deps[0].name, "serde_derive");
573        assert!(entry.deps[0].optional);
574        assert!(!entry.yanked);
575    }
576}
upstream_ontologist/providers/rust.rs

upstream_ontologist/providers/
rust.rs
