provenant/parsers/
readme.rs

1// SPDX-FileCopyrightText: nexB Inc. and others
2// ScanCode is a trademark of nexB Inc.
3// SPDX-FileCopyrightText: Provenant contributors
4// SPDX-License-Identifier: Apache-2.0
5// Derived from ScanCode Toolkit (Apache-2.0); modified. See NOTICE.
6
7//! Parser for third-party attribution README files.
8//!
9//! Extracts package metadata from semi-structured README files used to document
10//! third-party dependencies in Android, Chromium, Facebook, Google, and similar codebases.
11//!
12//! # Supported Formats
13//! - README.android
14//! - README.chromium
15//! - README.facebook
16//! - README.google
17//! - README.thirdparty
18//!
19//! # Key Features
20//! - Key:value pair extraction (both `:` and `=` separators)
21//! - Parent directory name fallback for packages without explicit names
22//! - Field name mapping to standardized PackageData fields
23//!
24//! # Implementation Notes
25//! - Keys are matched case-insensitively
26//! - Lines without valid separators are skipped
27//! - Multiple URL-related keys map to homepage_url (repo, source, upstream, etc.)
28//! - Separator precedence: the first separator (`:` or `=`) on each line is used
29
30use crate::models::PackageData;
31use crate::models::{DatasourceId, PackageType};
32use crate::parser_warn as warn;
33use crate::parsers::utils::{CappedIterExt, read_file_to_string, truncate_field};
34use std::path::Path;
35
36use super::PackageParser;
37use super::metadata::ParserMetadata;
38
39/// README attribution file parser.
40///
41/// Extracts package metadata from semi-structured README files commonly used
42/// to document third-party dependencies in large codebases.
43pub struct ReadmeParser;
44
45impl PackageParser for ReadmeParser {
46    const PACKAGE_TYPE: PackageType = PackageType::Readme;
47
48    fn metadata() -> Vec<ParserMetadata> {
49        vec![ParserMetadata {
50            description: "Third-party attribution README files",
51            file_patterns: &[
52                "**/README.android",
53                "**/README.chromium",
54                "**/README.facebook",
55                "**/README.google",
56                "**/README.thirdparty",
57            ],
58            package_type: "readme",
59            primary_language: "",
60            documentation_url: Some(
61                "https://github.com/chromium/chromium/blob/main/docs/contributing.md#third_party-components",
62            ),
63        }]
64    }
65
66    fn is_match(path: &Path) -> bool {
67        path.file_name().is_some_and(|name| {
68            let name = name.to_string_lossy().to_lowercase();
69            matches!(
70                name.as_str(),
71                "readme.android"
72                    | "readme.chromium"
73                    | "readme.facebook"
74                    | "readme.google"
75                    | "readme.thirdparty"
76            )
77        })
78    }
79
80    fn extract_packages(path: &Path) -> Vec<PackageData> {
81        let content = match read_file_to_string(path, None) {
82            Ok(content) => content,
83            Err(e) => {
84                warn!("Failed to read README file at {:?}: {}", path, e);
85                return vec![default_package_data()];
86            }
87        };
88
89        let mut pkg = default_package_data();
90
91        // Parse key:value pairs
92        for line in content.lines().capped("readme key:value lines") {
93            let line = line.trim();
94            if line.is_empty() {
95                continue;
96            }
97
98            let split_colon = line.split_once(':');
99            let split_equals = line.split_once('=');
100
101            let (key, value) = match (split_colon, split_equals) {
102                (Some((ck, cv)), Some((ek, _))) if ck.len() <= ek.len() => (ck.trim(), cv.trim()),
103                (_, Some((ek, ev))) => (ek.trim(), ev.trim()),
104                (Some((ck, cv)), None) => (ck.trim(), cv.trim()),
105                (None, None) => continue,
106            };
107
108            if key.is_empty() || value.is_empty() {
109                continue;
110            }
111
112            // Map README field to PackageData field (case-insensitive)
113            let key_lower = key.to_lowercase();
114            match key_lower.as_str() {
115                "name" | "project" => {
116                    pkg.name = Some(truncate_field(value.to_string()));
117                }
118                "version" => {
119                    pkg.version = Some(truncate_field(value.to_string()));
120                }
121                "copyright" => {
122                    pkg.copyright = Some(truncate_field(value.to_string()));
123                }
124                "download link" | "downloaded from" => {
125                    pkg.download_url = Some(truncate_field(value.to_string()));
126                }
127                "homepage" | "website" | "repo" | "source" | "upstream" | "url" | "project url" => {
128                    pkg.homepage_url = Some(truncate_field(value.to_string()));
129                }
130                "licence" | "license" => {
131                    pkg.extracted_license_statement = Some(truncate_field(value.to_string()));
132                }
133                _ => {
134                    // Unrecognized field, skip
135                }
136            }
137        }
138
139        // Fallback: use parent directory name if no name was found
140        if pkg.name.is_none()
141            && let Some(parent) = path.parent()
142            && let Some(parent_name) = parent.file_name()
143        {
144            pkg.name = Some(truncate_field(parent_name.to_string_lossy().to_string()));
145        }
146
147        vec![pkg]
148    }
149}
150
151fn default_package_data() -> PackageData {
152    PackageData {
153        package_type: Some(ReadmeParser::PACKAGE_TYPE),
154        datasource_id: Some(DatasourceId::Readme),
155        ..Default::default()
156    }
157}
158
159#[cfg(test)]
160mod tests {
161    use super::*;
162    use std::path::PathBuf;
163
164    #[test]
165    fn test_is_match_android() {
166        let valid = PathBuf::from("/some/path/README.android");
167        assert!(ReadmeParser::is_match(&valid));
168    }
169
170    #[test]
171    fn test_is_match_chromium() {
172        let valid = PathBuf::from("/some/path/README.chromium");
173        assert!(ReadmeParser::is_match(&valid));
174    }
175
176    #[test]
177    fn test_is_match_facebook() {
178        let valid = PathBuf::from("/some/path/README.facebook");
179        assert!(ReadmeParser::is_match(&valid));
180    }
181
182    #[test]
183    fn test_is_match_google() {
184        let valid = PathBuf::from("/some/path/README.google");
185        assert!(ReadmeParser::is_match(&valid));
186    }
187
188    #[test]
189    fn test_is_match_thirdparty() {
190        let valid = PathBuf::from("/some/path/README.thirdparty");
191        assert!(ReadmeParser::is_match(&valid));
192    }
193
194    #[test]
195    fn test_is_match_case_insensitive() {
196        let upper = PathBuf::from("/some/path/README.CHROMIUM");
197        let mixed = PathBuf::from("/some/path/README.ChRoMiUm");
198        assert!(ReadmeParser::is_match(&upper));
199        assert!(ReadmeParser::is_match(&mixed));
200    }
201
202    #[test]
203    fn test_is_match_negative_cases() {
204        let readme_md = PathBuf::from("/some/path/README.md");
205        let readme_txt = PathBuf::from("/some/path/README.txt");
206        let readme = PathBuf::from("/some/path/README");
207        let other = PathBuf::from("/some/path/INSTALL.txt");
208
209        assert!(!ReadmeParser::is_match(&readme_md));
210        assert!(!ReadmeParser::is_match(&readme_txt));
211        assert!(!ReadmeParser::is_match(&readme));
212        assert!(!ReadmeParser::is_match(&other));
213    }
214
215    #[test]
216    fn test_extract_chromium_format() {
217        let path = PathBuf::from("testdata/readme/chromium/third_party/example/README.chromium");
218        let pkg = ReadmeParser::extract_first_package(&path);
219
220        assert_eq!(pkg.package_type, Some(PackageType::Readme));
221        assert_eq!(pkg.name, Some("Example Library".to_string()));
222        assert_eq!(pkg.version, Some("2.1.0".to_string()));
223        assert_eq!(pkg.homepage_url, Some("https://example.com".to_string()));
224        assert_eq!(pkg.extracted_license_statement, Some("MIT".to_string()));
225        assert_eq!(pkg.datasource_id, Some(DatasourceId::Readme));
226    }
227
228    #[test]
229    fn test_extract_android_format() {
230        let path = PathBuf::from("testdata/readme/android/third_party/example/README.android");
231        let pkg = ReadmeParser::extract_first_package(&path);
232
233        assert_eq!(pkg.name, Some("Android Example".to_string()));
234        assert_eq!(pkg.version, Some("1.0".to_string()));
235        assert_eq!(
236            pkg.homepage_url,
237            Some("https://android.example.com".to_string())
238        );
239        assert_eq!(pkg.copyright, Some("2024 Google Inc.".to_string()));
240    }
241
242    #[test]
243    fn test_extract_facebook_format() {
244        let path = PathBuf::from("testdata/readme/facebook/third_party/example/README.facebook");
245        let pkg = ReadmeParser::extract_first_package(&path);
246
247        assert_eq!(pkg.name, Some("FB Library".to_string()));
248        assert_eq!(
249            pkg.download_url,
250            Some("https://github.com/example/fb-lib".to_string())
251        );
252        assert_eq!(
253            pkg.extracted_license_statement,
254            Some("BSD-3-Clause".to_string())
255        );
256    }
257
258    #[test]
259    fn test_extract_parent_dir_fallback() {
260        let path = PathBuf::from("testdata/readme/no-name/third_party/mylib/README.thirdparty");
261        let pkg = ReadmeParser::extract_first_package(&path);
262
263        // Should use parent directory name "mylib" since no name field in file
264        assert_eq!(pkg.name, Some("mylib".to_string()));
265        assert_eq!(pkg.homepage_url, Some("https://example.com".to_string()));
266        assert_eq!(pkg.version, Some("3.0".to_string()));
267    }
268
269    #[test]
270    fn test_extract_equals_separator() {
271        let path =
272            PathBuf::from("testdata/readme/equals-separator/third_party/eqlib/README.google");
273        let pkg = ReadmeParser::extract_first_package(&path);
274
275        assert_eq!(pkg.name, Some("Google Lib".to_string()));
276        assert_eq!(
277            pkg.homepage_url,
278            Some("https://google.example.com".to_string())
279        );
280        assert_eq!(
281            pkg.extracted_license_statement,
282            Some("Apache-2.0".to_string())
283        );
284    }
285
286    #[test]
287    fn test_case_insensitive_field_names() {
288        let path = PathBuf::from("testdata/readme/chromium/third_party/example/README.chromium");
289        let pkg = ReadmeParser::extract_first_package(&path);
290
291        // The test file uses "Name:", "URL:", "Version:", "License:"
292        // All should be recognized despite capitalization
293        assert!(pkg.name.is_some());
294        assert!(pkg.version.is_some());
295        assert!(pkg.homepage_url.is_some());
296        assert!(pkg.extracted_license_statement.is_some());
297    }
298
299    #[test]
300    fn test_invalid_file() {
301        let nonexistent = PathBuf::from("testdata/readme/nonexistent/README.chromium");
302        let pkg = ReadmeParser::extract_first_package(&nonexistent);
303
304        // Should return default data with proper type and datasource
305        assert_eq!(pkg.package_type, Some(PackageType::Readme));
306        assert_eq!(pkg.datasource_id, Some(DatasourceId::Readme));
307    }
308}
provenant/parsers/readme.rs

provenant/parsers/
readme.rs
