provenant/parsers/
readme.rs

1// SPDX-FileCopyrightText: nexB Inc. and others
2// SPDX-FileCopyrightText: Provenant contributors
3// SPDX-License-Identifier: Apache-2.0
4// Derived from ScanCode Toolkit (Apache-2.0); modified. See NOTICE.
5
6//! Parser for third-party attribution README files.
7//!
8//! Extracts package metadata from semi-structured README files used to document
9//! third-party dependencies in Android, Chromium, Facebook, Google, and similar codebases.
10//!
11//! # Supported Formats
12//! - README.android
13//! - README.chromium
14//! - README.facebook
15//! - README.google
16//! - README.thirdparty
17//!
18//! # Key Features
19//! - Key:value pair extraction (both `:` and `=` separators)
20//! - Parent directory name fallback for packages without explicit names
21//! - Field name mapping to standardized PackageData fields
22//!
23//! # Implementation Notes
24//! - Keys are matched case-insensitively
25//! - Lines without valid separators are skipped
26//! - Multiple URL-related keys map to homepage_url (repo, source, upstream, etc.)
27//! - Separator precedence: the first separator (`:` or `=`) on each line is used
28
29use crate::models::PackageData;
30use crate::models::{DatasourceId, PackageType};
31use crate::parser_warn as warn;
32use crate::parsers::utils::{CappedIterExt, read_file_to_string, truncate_field};
33use std::path::Path;
34
35use super::PackageParser;
36use super::metadata::ParserMetadata;
37
38/// README attribution file parser.
39///
40/// Extracts package metadata from semi-structured README files commonly used
41/// to document third-party dependencies in large codebases.
42pub struct ReadmeParser;
43
44impl PackageParser for ReadmeParser {
45    const PACKAGE_TYPE: PackageType = PackageType::Readme;
46
47    fn metadata() -> Vec<ParserMetadata> {
48        vec![ParserMetadata {
49            description: "Third-party attribution README files",
50            file_patterns: &[
51                "**/README.android",
52                "**/README.chromium",
53                "**/README.facebook",
54                "**/README.google",
55                "**/README.thirdparty",
56            ],
57            package_type: "readme",
58            primary_language: "",
59            documentation_url: Some(
60                "https://github.com/chromium/chromium/blob/main/docs/contributing.md#third_party-components",
61            ),
62        }]
63    }
64
65    fn is_match(path: &Path) -> bool {
66        path.file_name().is_some_and(|name| {
67            let name = name.to_string_lossy().to_lowercase();
68            matches!(
69                name.as_str(),
70                "readme.android"
71                    | "readme.chromium"
72                    | "readme.facebook"
73                    | "readme.google"
74                    | "readme.thirdparty"
75            )
76        })
77    }
78
79    fn extract_packages(path: &Path) -> Vec<PackageData> {
80        let content = match read_file_to_string(path, None) {
81            Ok(content) => content,
82            Err(e) => {
83                warn!("Failed to read README file at {:?}: {}", path, e);
84                return vec![default_package_data()];
85            }
86        };
87
88        let mut pkg = default_package_data();
89
90        // Parse key:value pairs
91        for line in content.lines().capped("readme key:value lines") {
92            let line = line.trim();
93            if line.is_empty() {
94                continue;
95            }
96
97            let split_colon = line.split_once(':');
98            let split_equals = line.split_once('=');
99
100            let (key, value) = match (split_colon, split_equals) {
101                (Some((ck, cv)), Some((ek, _))) if ck.len() <= ek.len() => (ck.trim(), cv.trim()),
102                (_, Some((ek, ev))) => (ek.trim(), ev.trim()),
103                (Some((ck, cv)), None) => (ck.trim(), cv.trim()),
104                (None, None) => continue,
105            };
106
107            if key.is_empty() || value.is_empty() {
108                continue;
109            }
110
111            // Map README field to PackageData field (case-insensitive)
112            let key_lower = key.to_lowercase();
113            match key_lower.as_str() {
114                "name" | "project" => {
115                    pkg.name = Some(truncate_field(value.to_string()));
116                }
117                "version" => {
118                    pkg.version = Some(truncate_field(value.to_string()));
119                }
120                "copyright" => {
121                    pkg.copyright = Some(truncate_field(value.to_string()));
122                }
123                "download link" | "downloaded from" => {
124                    pkg.download_url = Some(truncate_field(value.to_string()));
125                }
126                "homepage" | "website" | "repo" | "source" | "upstream" | "url" | "project url" => {
127                    pkg.homepage_url = Some(truncate_field(value.to_string()));
128                }
129                "licence" | "license" => {
130                    pkg.extracted_license_statement = Some(truncate_field(value.to_string()));
131                }
132                _ => {
133                    // Unrecognized field, skip
134                }
135            }
136        }
137
138        // Fallback: use parent directory name if no name was found
139        if pkg.name.is_none()
140            && let Some(parent) = path.parent()
141            && let Some(parent_name) = parent.file_name()
142        {
143            pkg.name = Some(truncate_field(parent_name.to_string_lossy().to_string()));
144        }
145
146        vec![pkg]
147    }
148}
149
150fn default_package_data() -> PackageData {
151    PackageData {
152        package_type: Some(ReadmeParser::PACKAGE_TYPE),
153        datasource_id: Some(DatasourceId::Readme),
154        ..Default::default()
155    }
156}
157
158#[cfg(test)]
159mod tests {
160    use super::*;
161    use std::path::PathBuf;
162
163    #[test]
164    fn test_is_match_android() {
165        let valid = PathBuf::from("/some/path/README.android");
166        assert!(ReadmeParser::is_match(&valid));
167    }
168
169    #[test]
170    fn test_is_match_chromium() {
171        let valid = PathBuf::from("/some/path/README.chromium");
172        assert!(ReadmeParser::is_match(&valid));
173    }
174
175    #[test]
176    fn test_is_match_facebook() {
177        let valid = PathBuf::from("/some/path/README.facebook");
178        assert!(ReadmeParser::is_match(&valid));
179    }
180
181    #[test]
182    fn test_is_match_google() {
183        let valid = PathBuf::from("/some/path/README.google");
184        assert!(ReadmeParser::is_match(&valid));
185    }
186
187    #[test]
188    fn test_is_match_thirdparty() {
189        let valid = PathBuf::from("/some/path/README.thirdparty");
190        assert!(ReadmeParser::is_match(&valid));
191    }
192
193    #[test]
194    fn test_is_match_case_insensitive() {
195        let upper = PathBuf::from("/some/path/README.CHROMIUM");
196        let mixed = PathBuf::from("/some/path/README.ChRoMiUm");
197        assert!(ReadmeParser::is_match(&upper));
198        assert!(ReadmeParser::is_match(&mixed));
199    }
200
201    #[test]
202    fn test_is_match_negative_cases() {
203        let readme_md = PathBuf::from("/some/path/README.md");
204        let readme_txt = PathBuf::from("/some/path/README.txt");
205        let readme = PathBuf::from("/some/path/README");
206        let other = PathBuf::from("/some/path/INSTALL.txt");
207
208        assert!(!ReadmeParser::is_match(&readme_md));
209        assert!(!ReadmeParser::is_match(&readme_txt));
210        assert!(!ReadmeParser::is_match(&readme));
211        assert!(!ReadmeParser::is_match(&other));
212    }
213
214    #[test]
215    fn test_extract_chromium_format() {
216        let path = PathBuf::from("testdata/readme/chromium/third_party/example/README.chromium");
217        let pkg = ReadmeParser::extract_first_package(&path);
218
219        assert_eq!(pkg.package_type, Some(PackageType::Readme));
220        assert_eq!(pkg.name, Some("Example Library".to_string()));
221        assert_eq!(pkg.version, Some("2.1.0".to_string()));
222        assert_eq!(pkg.homepage_url, Some("https://example.com".to_string()));
223        assert_eq!(pkg.extracted_license_statement, Some("MIT".to_string()));
224        assert_eq!(pkg.datasource_id, Some(DatasourceId::Readme));
225    }
226
227    #[test]
228    fn test_extract_android_format() {
229        let path = PathBuf::from("testdata/readme/android/third_party/example/README.android");
230        let pkg = ReadmeParser::extract_first_package(&path);
231
232        assert_eq!(pkg.name, Some("Android Example".to_string()));
233        assert_eq!(pkg.version, Some("1.0".to_string()));
234        assert_eq!(
235            pkg.homepage_url,
236            Some("https://android.example.com".to_string())
237        );
238        assert_eq!(pkg.copyright, Some("2024 Google Inc.".to_string()));
239    }
240
241    #[test]
242    fn test_extract_facebook_format() {
243        let path = PathBuf::from("testdata/readme/facebook/third_party/example/README.facebook");
244        let pkg = ReadmeParser::extract_first_package(&path);
245
246        assert_eq!(pkg.name, Some("FB Library".to_string()));
247        assert_eq!(
248            pkg.download_url,
249            Some("https://github.com/example/fb-lib".to_string())
250        );
251        assert_eq!(
252            pkg.extracted_license_statement,
253            Some("BSD-3-Clause".to_string())
254        );
255    }
256
257    #[test]
258    fn test_extract_parent_dir_fallback() {
259        let path = PathBuf::from("testdata/readme/no-name/third_party/mylib/README.thirdparty");
260        let pkg = ReadmeParser::extract_first_package(&path);
261
262        // Should use parent directory name "mylib" since no name field in file
263        assert_eq!(pkg.name, Some("mylib".to_string()));
264        assert_eq!(pkg.homepage_url, Some("https://example.com".to_string()));
265        assert_eq!(pkg.version, Some("3.0".to_string()));
266    }
267
268    #[test]
269    fn test_extract_equals_separator() {
270        let path =
271            PathBuf::from("testdata/readme/equals-separator/third_party/eqlib/README.google");
272        let pkg = ReadmeParser::extract_first_package(&path);
273
274        assert_eq!(pkg.name, Some("Google Lib".to_string()));
275        assert_eq!(
276            pkg.homepage_url,
277            Some("https://google.example.com".to_string())
278        );
279        assert_eq!(
280            pkg.extracted_license_statement,
281            Some("Apache-2.0".to_string())
282        );
283    }
284
285    #[test]
286    fn test_case_insensitive_field_names() {
287        let path = PathBuf::from("testdata/readme/chromium/third_party/example/README.chromium");
288        let pkg = ReadmeParser::extract_first_package(&path);
289
290        // The test file uses "Name:", "URL:", "Version:", "License:"
291        // All should be recognized despite capitalization
292        assert!(pkg.name.is_some());
293        assert!(pkg.version.is_some());
294        assert!(pkg.homepage_url.is_some());
295        assert!(pkg.extracted_license_statement.is_some());
296    }
297
298    #[test]
299    fn test_invalid_file() {
300        let nonexistent = PathBuf::from("testdata/readme/nonexistent/README.chromium");
301        let pkg = ReadmeParser::extract_first_package(&nonexistent);
302
303        // Should return default data with proper type and datasource
304        assert_eq!(pkg.package_type, Some(PackageType::Readme));
305        assert_eq!(pkg.datasource_id, Some(DatasourceId::Readme));
306    }
307}
provenant/parsers/readme.rs

provenant/parsers/
readme.rs
