upstream_ontologist/providers/
perl.rs

1use crate::{
2    Certainty, GuesserSettings, Origin, ProviderError, UpstreamDatum, UpstreamDatumWithMetadata,
3    UpstreamMetadata,
4};
5use lazy_regex::regex;
6use serde::Deserialize;
7
8use std::collections::HashMap;
9use std::fs::File;
10use std::io::{BufRead, BufReader, Read};
11use std::path::{Path, PathBuf};
12use std::process::Command;
13
14/// Extracts upstream metadata from Perl POD documentation
15pub fn guess_from_pod(
16    contents: &str,
17    origin: &Origin,
18) -> std::result::Result<Vec<UpstreamDatumWithMetadata>, ProviderError> {
19    let mut by_header: HashMap<String, String> = HashMap::new();
20    let mut inheader: Option<String> = None;
21
22    for line in contents.lines() {
23        if line.starts_with("=head1 ") {
24            inheader = Some(line.trim_start_matches("=head1 ").to_string());
25            by_header.insert(inheader.clone().unwrap().to_uppercase(), String::new());
26        } else if let Some(header) = &inheader {
27            if let Some(value) = by_header.get_mut(&header.to_uppercase()) {
28                value.push_str(line)
29            }
30        }
31    }
32
33    let mut upstream_data: Vec<UpstreamDatumWithMetadata> = Vec::new();
34
35    if let Some(description) = by_header.get("DESCRIPTION") {
36        let mut description = description.trim_start_matches('\n').to_string();
37        description = regex!(r"[FXZSCBI]\\<([^>]+)>")
38            .replace_all(&description, "$1")
39            .into_owned();
40        description = regex!(r"L\\<([^\|]+)\|([^\\>]+)\\>")
41            .replace_all(&description, "$2")
42            .into_owned();
43        description = regex!(r"L\\<([^\\>]+)\\>")
44            .replace_all(&description, "$1")
45            .into_owned();
46
47        upstream_data.push(UpstreamDatumWithMetadata {
48            datum: UpstreamDatum::Description(description),
49            certainty: Some(Certainty::Certain),
50            origin: Some(origin.clone()),
51        });
52    }
53
54    if let Some(name) = by_header.get("NAME") {
55        let lines: Vec<&str> = name.trim().lines().collect();
56        if let Some(line) = lines.first() {
57            if let Some((name, summary)) = line.split_once(" - ") {
58                upstream_data.push(UpstreamDatumWithMetadata {
59                    datum: UpstreamDatum::Name(name.trim().to_string()),
60                    certainty: Some(Certainty::Confident),
61                    origin: Some(origin.clone()),
62                });
63                upstream_data.push(UpstreamDatumWithMetadata {
64                    datum: UpstreamDatum::Summary(summary.trim().to_string()),
65                    certainty: Some(Certainty::Confident),
66                    origin: Some(origin.clone()),
67                });
68            } else if !line.contains(' ') {
69                upstream_data.push(UpstreamDatumWithMetadata {
70                    datum: UpstreamDatum::Name(line.trim().to_string()),
71                    certainty: Some(Certainty::Confident),
72                    origin: Some(origin.clone()),
73                });
74            }
75        }
76    }
77
78    Ok(upstream_data)
79}
80
81/// Extracts upstream metadata from a Perl module file
82pub fn guess_from_perl_module(
83    path: &Path,
84) -> std::result::Result<Vec<UpstreamDatumWithMetadata>, ProviderError> {
85    match Command::new("perldoc").arg("-u").arg(path).output() {
86        Ok(output) => guess_from_pod(
87            &String::from_utf8_lossy(&output.stdout),
88            &Origin::Path(path.into()),
89        ),
90        Err(e) => Err(ProviderError::Other(format!(
91            "Error running perldoc: {}",
92            e
93        ))),
94    }
95}
96
97/// Guesses upstream metadata based on Perl distribution name
98pub fn guess_from_perl_dist_name(
99    path: &Path,
100    dist_name: &str,
101) -> std::result::Result<Vec<UpstreamDatumWithMetadata>, ProviderError> {
102    let mod_path = PathBuf::from(format!(
103        "{}/lib/{}.pm",
104        std::path::Path::new(path)
105            .parent()
106            .expect("parent")
107            .display(),
108        dist_name.replace('-', "/")
109    ));
110
111    if mod_path.exists() {
112        guess_from_perl_module(mod_path.as_path())
113    } else {
114        Ok(Vec::new())
115    }
116}
117
118#[cfg(feature = "dist-ini")]
119/// Extracts upstream metadata from Dist::Zilla dist.ini file
120pub fn guess_from_dist_ini(
121    path: &Path,
122    _settings: &GuesserSettings,
123) -> std::result::Result<Vec<UpstreamDatumWithMetadata>, ProviderError> {
124    let parser = ini::Ini::load_from_file(path)
125        .map_err(|e| ProviderError::ParseError(format!("Error parsing dist.ini: {}", e)))?;
126
127    let dist_name = parser
128        .get_from::<&str>(None, "name")
129        .map(|name| UpstreamDatumWithMetadata {
130            datum: UpstreamDatum::Name(name.to_string()),
131            certainty: Some(Certainty::Certain),
132            origin: Some(path.into()),
133        });
134
135    let version =
136        parser
137            .get_from::<&str>(None, "version")
138            .map(|version| UpstreamDatumWithMetadata {
139                datum: UpstreamDatum::Version(version.to_string()),
140                certainty: Some(Certainty::Certain),
141                origin: Some(path.into()),
142            });
143
144    let summary =
145        parser
146            .get_from::<&str>(None, "abstract")
147            .map(|summary| UpstreamDatumWithMetadata {
148                datum: UpstreamDatum::Summary(summary.to_string()),
149                certainty: Some(Certainty::Certain),
150                origin: Some(path.into()),
151            });
152
153    let bug_database = parser
154        .get_from(Some("MetaResources"), "bugtracker.web")
155        .map(|bugtracker| UpstreamDatumWithMetadata {
156            datum: UpstreamDatum::BugDatabase(bugtracker.to_string()),
157            certainty: Some(Certainty::Certain),
158            origin: Some(path.into()),
159        });
160
161    let repository = parser
162        .get_from(Some("MetaResources"), "repository.url")
163        .map(|repository| UpstreamDatumWithMetadata {
164            datum: UpstreamDatum::Repository(repository.to_string()),
165            certainty: Some(Certainty::Certain),
166            origin: Some(path.into()),
167        });
168
169    let license =
170        parser
171            .get_from::<&str>(None, "license")
172            .map(|license| UpstreamDatumWithMetadata {
173                datum: UpstreamDatum::License(license.to_string()),
174                certainty: Some(Certainty::Certain),
175                origin: Some(path.into()),
176            });
177
178    let copyright = match (
179        parser.get_from::<&str>(None, "copyright_year"),
180        parser.get_from::<&str>(None, "copyright_holder"),
181    ) {
182        (Some(year), Some(holder)) => Some(UpstreamDatumWithMetadata {
183            datum: UpstreamDatum::Copyright(format!("{} {}", year, holder)),
184            certainty: Some(Certainty::Certain),
185            origin: Some(path.into()),
186        }),
187        _ => None,
188    };
189
190    let mut upstream_data: Vec<UpstreamDatumWithMetadata> = Vec::new();
191
192    if let Some(dist_name) = dist_name {
193        upstream_data.push(dist_name);
194    }
195    if let Some(version) = version {
196        upstream_data.push(version);
197    }
198    if let Some(summary) = summary {
199        upstream_data.push(summary);
200    }
201    if let Some(bug_database) = bug_database {
202        upstream_data.push(bug_database);
203    }
204    if let Some(repository) = repository {
205        upstream_data.push(repository);
206    }
207    if let Some(license) = license {
208        upstream_data.push(license);
209    }
210    if let Some(copyright) = copyright {
211        upstream_data.push(copyright);
212    }
213
214    if let Some(dist_name) = parser.get_from::<&str>(None, "name") {
215        upstream_data.extend(guess_from_perl_dist_name(path, dist_name)?);
216    }
217
218    Ok(upstream_data)
219}
220
221/// Extracts upstream metadata from Perl META.json file
222pub fn guess_from_meta_json(
223    path: &Path,
224    _settings: &GuesserSettings,
225) -> std::result::Result<Vec<UpstreamDatumWithMetadata>, ProviderError> {
226    let mut file = File::open(path)?;
227    let mut contents = String::new();
228    file.read_to_string(&mut contents)?;
229
230    let data: serde_json::Map<String, serde_json::Value> = serde_json::from_str(&contents)
231        .map_err(|e| ProviderError::ParseError(format!("Error parsing META.json: {}", e)))?;
232
233    let mut upstream_data: Vec<UpstreamDatumWithMetadata> = Vec::new();
234
235    if let Some(name) = data.get("name").and_then(serde_json::Value::as_str) {
236        upstream_data.push(UpstreamDatumWithMetadata {
237            datum: UpstreamDatum::Name(name.to_string()),
238            certainty: Some(Certainty::Certain),
239            origin: Some(path.into()),
240        });
241    }
242
243    if let Some(version) = data.get("version").and_then(serde_json::Value::as_str) {
244        let version = version.strip_prefix('v').unwrap_or(version);
245        upstream_data.push(UpstreamDatumWithMetadata {
246            datum: UpstreamDatum::Version(version.to_string()),
247            certainty: Some(Certainty::Certain),
248            origin: Some(path.into()),
249        });
250    }
251
252    if let Some(summary) = data.get("abstract").and_then(serde_json::Value::as_str) {
253        upstream_data.push(UpstreamDatumWithMetadata {
254            datum: UpstreamDatum::Summary(summary.to_string()),
255            certainty: Some(Certainty::Certain),
256            origin: Some(path.into()),
257        });
258    }
259
260    if let Some(resources) = data.get("resources").and_then(serde_json::Value::as_object) {
261        if let Some(bugtracker) = resources
262            .get("bugtracker")
263            .and_then(serde_json::Value::as_object)
264        {
265            if let Some(web) = bugtracker.get("web").and_then(serde_json::Value::as_str) {
266                upstream_data.push(UpstreamDatumWithMetadata {
267                    datum: UpstreamDatum::BugDatabase(web.to_string()),
268                    certainty: Some(Certainty::Certain),
269                    origin: Some(path.into()),
270                });
271                // TODO: Support resources["bugtracker"]["mailto"]
272            }
273        }
274
275        if let Some(homepage) = resources
276            .get("homepage")
277            .and_then(serde_json::Value::as_str)
278        {
279            upstream_data.push(UpstreamDatumWithMetadata {
280                datum: UpstreamDatum::Homepage(homepage.to_string()),
281                certainty: Some(Certainty::Certain),
282                origin: Some(path.into()),
283            });
284        }
285
286        if let Some(repo) = resources
287            .get("repository")
288            .and_then(serde_json::Value::as_object)
289        {
290            if let Some(url) = repo.get("url").and_then(serde_json::Value::as_str) {
291                upstream_data.push(UpstreamDatumWithMetadata {
292                    datum: UpstreamDatum::Repository(url.to_string()),
293                    certainty: Some(Certainty::Certain),
294                    origin: Some(path.into()),
295                });
296            }
297
298            if let Some(web) = repo.get("web").and_then(serde_json::Value::as_str) {
299                upstream_data.push(UpstreamDatumWithMetadata {
300                    datum: UpstreamDatum::RepositoryBrowse(web.to_string()),
301                    certainty: Some(Certainty::Certain),
302                    origin: Some(path.into()),
303                });
304            }
305        }
306    }
307
308    // Wild guess:
309    if let Some(dist_name) = data.get("name").and_then(serde_json::Value::as_str) {
310        upstream_data.extend(guess_from_perl_dist_name(path, dist_name)?);
311    }
312
313    Ok(upstream_data)
314}
315
316/// Guess upstream metadata from a META.yml file.
317///
318/// See <http://module-build.sourceforge.net/META-spec-v1.4.html> for the
319/// specification of the format.
320pub fn guess_from_meta_yml(
321    path: &Path,
322    _settings: &GuesserSettings,
323) -> std::result::Result<Vec<UpstreamDatumWithMetadata>, ProviderError> {
324    let mut file = File::open(path)?;
325
326    let mut contents = String::new();
327    file.read_to_string(&mut contents)?;
328
329    let data: serde_norway::Value = serde_norway::from_str(&contents)
330        .map_err(|e| ProviderError::ParseError(format!("Error parsing META.yml: {}", e)))?;
331
332    let mut upstream_data = Vec::new();
333
334    if let Some(name) = data.get("name") {
335        if let Some(name) = name.as_str() {
336            upstream_data.push(UpstreamDatumWithMetadata {
337                datum: UpstreamDatum::Name(name.to_string()),
338                certainty: Some(Certainty::Certain),
339                origin: Some(path.into()),
340            });
341        }
342    }
343
344    if let Some(license) = data.get("license") {
345        if let Some(license) = license.as_str() {
346            upstream_data.push(UpstreamDatumWithMetadata {
347                datum: UpstreamDatum::License(license.to_string()),
348                certainty: Some(Certainty::Certain),
349                origin: Some(path.into()),
350            });
351        }
352    }
353
354    if let Some(version) = data.get("version") {
355        if let Some(version) = version.as_str() {
356            upstream_data.push(UpstreamDatumWithMetadata {
357                datum: UpstreamDatum::Version(version.to_string()),
358                certainty: Some(Certainty::Certain),
359                origin: Some(path.into()),
360            });
361        }
362    }
363
364    if let Some(resources) = data.get("resources") {
365        if let Some(bugtracker) = resources.get("bugtracker").and_then(|v| v.as_str()) {
366            upstream_data.push(UpstreamDatumWithMetadata {
367                datum: UpstreamDatum::BugDatabase(bugtracker.to_string()),
368                certainty: Some(Certainty::Certain),
369                origin: Some(path.into()),
370            });
371        }
372
373        if let Some(homepage) = resources.get("homepage").and_then(|v| v.as_str()) {
374            upstream_data.push(UpstreamDatumWithMetadata {
375                datum: UpstreamDatum::Homepage(homepage.to_string()),
376                certainty: Some(Certainty::Certain),
377                origin: Some(path.into()),
378            });
379        }
380
381        if let Some(repository) = resources.get("repository") {
382            if let Some(url) = repository.get("url").and_then(|v| v.as_str()) {
383                upstream_data.push(UpstreamDatumWithMetadata {
384                    datum: UpstreamDatum::Repository(url.to_string()),
385                    certainty: Some(Certainty::Certain),
386                    origin: Some(path.into()),
387                });
388            } else if let Some(url) = repository.as_str() {
389                upstream_data.push(UpstreamDatumWithMetadata {
390                    datum: UpstreamDatum::Repository(url.to_string()),
391                    certainty: Some(Certainty::Certain),
392                    origin: Some(path.into()),
393                });
394            } else {
395                log::warn!(
396                    "META.yml: unexpected type for 'repository': {:?}",
397                    repository
398                );
399            }
400        }
401    }
402
403    // Wild guess:
404    if let Some(dist_name) = data.get("name") {
405        if let Some(dist_name) = dist_name.as_str() {
406            upstream_data.extend(guess_from_perl_dist_name(path, dist_name)?);
407        }
408    }
409
410    Ok(upstream_data)
411}
412
413/// Extracts upstream metadata from Makefile.PL
414pub fn guess_from_makefile_pl(
415    path: &Path,
416    _settings: &GuesserSettings,
417) -> std::result::Result<Vec<UpstreamDatumWithMetadata>, ProviderError> {
418    let mut dist_name = None;
419    let file = File::open(path)?;
420    let reader = BufReader::new(file);
421    let mut results = Vec::new();
422    let name_regex = regex!("name '([^'\"]+)';$");
423    let repository_regex = regex!("repository '([^'\"]+)';$");
424
425    for line in reader.lines().map_while(Result::ok) {
426        if let Some(captures) = name_regex.captures(&line) {
427            dist_name = Some(captures.get(1).unwrap().as_str().to_owned());
428            let name = dist_name.as_ref().unwrap().to_owned();
429            results.push(UpstreamDatumWithMetadata {
430                datum: UpstreamDatum::Name(name),
431                certainty: Some(Certainty::Confident),
432                origin: Some(path.into()),
433            });
434        }
435        if let Some(captures) = repository_regex.captures(&line) {
436            let repository = captures.get(1).unwrap().as_str().to_owned();
437            results.push(UpstreamDatumWithMetadata {
438                datum: UpstreamDatum::Repository(repository),
439                certainty: Some(Certainty::Confident),
440                origin: Some(path.into()),
441            });
442        }
443    }
444
445    if let Some(dist_name) = dist_name {
446        results.extend(guess_from_perl_dist_name(path, &dist_name)?);
447    }
448
449    Ok(results)
450}
451
452/// CPAN module information
453#[derive(Deserialize)]
454pub struct Module {
455    /// Numeric version representation
456    pub version_numified: f64,
457    /// Version string
458    pub version: String,
459    /// Whether the module is authorized
460    pub authorized: bool,
461    /// Module name
462    pub name: String,
463    /// Whether the module is indexed
464    pub indexed: bool,
465}
466
467/// File statistics for a CPAN module
468#[derive(Deserialize)]
469pub struct Stat {
470    /// User ID
471    pub uid: isize,
472    /// Modification time
473    pub mtime: isize,
474    /// File size in bytes
475    pub size: isize,
476    /// File mode
477    pub mode: isize,
478    /// Group ID
479    pub gid: isize,
480}
481
482/// Complete CPAN module metadata
483#[derive(Deserialize)]
484pub struct CpanModule {
485    /// Module maturity level
486    pub maturity: String,
487    /// Release identifier
488    pub release: String,
489    /// Author name
490    pub author: String,
491    /// Source lines of Perl code
492    pub slop: isize,
493    /// Download URL for the module
494    pub download_url: url::Url,
495    /// List of modules in this release
496    pub module: Vec<Module>,
497    /// POD documentation lines
498    pub pod_lines: Vec<String>,
499    /// Module version
500    pub version: String,
501    /// Whether the module is deprecated
502    pub deprecated: bool,
503    /// Nesting level
504    pub level: isize,
505    /// MIME type of the file
506    pub mime: String,
507    /// Release date
508    pub date: String,
509    /// File path within the distribution
510    pub path: String,
511    /// Distribution name
512    pub distribution: String,
513    /// POD documentation content
514    pub pod: String,
515    /// Module name
516    pub name: String,
517    /// Source lines of code
518    pub sloc: isize,
519    /// File statistics
520    pub stat: Stat,
521    /// Numeric version representation
522    pub version_numified: f64,
523    /// Whether the file is binary
524    pub binary: bool,
525    /// Unique identifier
526    pub id: String,
527    /// Whether this is a directory
528    pub directory: bool,
529    /// Whether the module is indexed
530    pub indexed: bool,
531    /// Whether the module is authorized
532    pub authorized: bool,
533}
534
535impl TryFrom<CpanModule> for UpstreamMetadata {
536    type Error = crate::ProviderError;
537
538    fn try_from(value: CpanModule) -> Result<Self, Self::Error> {
539        let mut metadata = UpstreamMetadata::default();
540
541        metadata.insert(UpstreamDatumWithMetadata {
542            datum: UpstreamDatum::Name(value.name),
543            certainty: Some(Certainty::Certain),
544            origin: None,
545        });
546
547        metadata.insert(UpstreamDatumWithMetadata {
548            datum: UpstreamDatum::Version(value.version),
549            certainty: Some(Certainty::Certain),
550            origin: None,
551        });
552
553        metadata.insert(UpstreamDatumWithMetadata {
554            datum: UpstreamDatum::Homepage(value.download_url.to_string()),
555            certainty: Some(Certainty::Certain),
556            origin: None,
557        });
558
559        metadata.insert(UpstreamDatumWithMetadata {
560            datum: UpstreamDatum::Download(value.download_url.to_string()),
561            certainty: Some(Certainty::Certain),
562            origin: None,
563        });
564
565        metadata.insert(UpstreamDatumWithMetadata {
566            datum: UpstreamDatum::Author(vec![crate::Person::from(value.author.as_str())]),
567            certainty: Some(Certainty::Certain),
568            origin: None,
569        });
570
571        Ok(metadata)
572    }
573}
574
575/// Loads CPAN module data from the MetaCPAN API
576pub async fn load_cpan_data(module: &str) -> Result<Option<CpanModule>, crate::ProviderError> {
577    let url: url::Url = format!("https://fastapi.metacpan.org/v1/release/{}", module)
578        .parse()
579        .map_err(|e: url::ParseError| crate::ProviderError::Other(e.to_string()))?;
580
581    let data = crate::load_json_url(&url, None).await?;
582
583    let cpan_module: CpanModule = serde_json::from_value(data).map_err(|e| {
584        crate::ProviderError::ParseError(format!("Failed to parse CPAN data: {}", e))
585    })?;
586    Ok(Some(cpan_module))
587}
588
589/// Retrieves upstream metadata for a Perl module from CPAN
590pub async fn remote_cpan_data(module: &str) -> Result<UpstreamMetadata, crate::ProviderError> {
591    let data = load_cpan_data(module).await?;
592
593    match data {
594        Some(data) => data.try_into(),
595        None => Ok(UpstreamMetadata::default()),
596    }
597}
598
599#[cfg(test)]
600mod tests {
601    use super::*;
602
603    #[test]
604    fn test_meta_yml_resources_wrong_types() {
605        let td = tempfile::tempdir().unwrap();
606        let path = td.path().join("META.yml");
607        std::fs::write(
608            &path,
609            r#"---
610name: Test-Module
611resources:
612  bugtracker: 123
613  homepage: true
614  repository:
615    - not
616    - a
617    - string
618"#,
619        )
620        .unwrap();
621        let result = super::guess_from_meta_yml(&path, &super::GuesserSettings::default()).unwrap();
622        // Name should parse, resources with wrong types should be skipped
623        assert!(result
624            .iter()
625            .any(|d| matches!(&d.datum, super::UpstreamDatum::Name(n) if n == "Test-Module")));
626        assert!(!result
627            .iter()
628            .any(|d| matches!(&d.datum, super::UpstreamDatum::BugDatabase(_))));
629        assert!(!result
630            .iter()
631            .any(|d| matches!(&d.datum, super::UpstreamDatum::Homepage(_))));
632        assert!(!result
633            .iter()
634            .any(|d| matches!(&d.datum, super::UpstreamDatum::Repository(_))));
635    }
636
637    #[test]
638    fn test_load_from_json() {
639        let text = include_str!("../testdata/cpan.json");
640        let cpan_module: CpanModule = serde_json::from_str(text).unwrap();
641
642        assert_eq!("Parse-Pidl-0.02", cpan_module.release);
643    }
644}
upstream_ontologist/providers/perl.rs

upstream_ontologist/providers/
perl.rs
