provenant/utils/
sourcemap.rs

1// SPDX-FileCopyrightText: nexB Inc. and others
2// ScanCode is a trademark of nexB Inc.
3// SPDX-FileCopyrightText: Provenant contributors
4// SPDX-License-Identifier: Apache-2.0
5// Derived from ScanCode Toolkit (Apache-2.0); modified. See NOTICE.
6
7//! Source map file processing for scanner text detection.
8//!
9//! Source map files (.js.map, .css.map) are JSON files containing embedded
10//! source code in a `sourcesContent` array. This module extracts that content
11//! so the scanner can detect licenses and parties from the embedded sources
12//! instead of from the raw source map JSON wrapper.
13
14use std::borrow::Cow;
15use std::path::Path;
16
17/// Check if a file is a source map file based on extension.
18pub fn is_sourcemap(path: &Path) -> bool {
19    path.file_name()
20        .and_then(|name| name.to_str())
21        .map(|name| {
22            let name_lower = name.to_lowercase();
23            name_lower.ends_with(".js.map") || name_lower.ends_with(".css.map")
24        })
25        .unwrap_or(false)
26}
27
28/// Extract source content from a source map JSON file.
29///
30/// Parses the JSON and extracts all entries from `sourcesContent`,
31/// combining them with newlines for license detection.
32///
33/// Returns `Some(combined_text)` if successfully parsed with content.
34/// Returns `None` if JSON parsing fails or no sourcesContent exists.
35pub fn extract_sourcemap_content(json_text: &str) -> Option<String> {
36    let json: serde_json::Value = serde_json::from_str(json_text).ok()?;
37    let sources = json.get("sourcesContent")?.as_array()?;
38
39    let combined: String = sources
40        .iter()
41        .filter_map(|v| v.as_str())
42        .map(replace_verbatim_cr_lf_chars)
43        .collect::<Vec<_>>()
44        .join("\n");
45
46    if combined.is_empty() {
47        None
48    } else {
49        Some(combined)
50    }
51}
52
53/// Return the text scanners should inspect for this file.
54///
55/// Source maps and Jupyter notebooks embed the human-authored text inside a JSON
56/// wrapper; for those we hand scanners the extracted text instead of the raw JSON
57/// so detection sees what a reader would. All other files pass through unchanged.
58pub fn detection_text<'a>(path: &Path, text: &'a str) -> Cow<'a, str> {
59    if is_sourcemap(path) {
60        return extract_sourcemap_content(text)
61            .map(Cow::Owned)
62            .unwrap_or(Cow::Borrowed(text));
63    }
64
65    if crate::utils::notebook::is_notebook(path) {
66        return crate::utils::notebook::extract_notebook_content(text)
67            .map(Cow::Owned)
68            .unwrap_or(Cow::Borrowed(text));
69    }
70
71    Cow::Borrowed(text)
72}
73
74/// Replace verbatim escaped CR/LF characters with actual newlines.
75///
76/// This matches Python's `replace_verbatim_cr_lf_chars()` behavior exactly:
77/// - Double-escaped (e.g., source had literal `\r` that was escaped again):
78///   - `\\r\\n` (backslash-backslash-r-backslash-backslash-n) → newline
79///   - `\\r` (backslash-backslash-r) → newline
80///   - `\\n` (backslash-backslash-n) → newline
81/// - Single-escaped (e.g., JSON-escaped newlines):
82///   - `\r\n` (backslash-r-backslash-n) → newline
83///   - `\r` (backslash-r) → newline
84///   - `\n` (backslash-n) → newline
85fn replace_verbatim_cr_lf_chars(s: &str) -> String {
86    s.replace("\\\\r\\\\n", "\n")
87        .replace("\\r\\n", "\n")
88        .replace("\\\\r", "\n")
89        .replace("\\\\n", "\n")
90        .replace("\\r", "\n")
91        .replace("\\n", "\n")
92}
93
94#[cfg(test)]
95mod tests {
96    use super::*;
97    use std::path::PathBuf;
98
99    #[test]
100    fn test_is_sourcemap_js_map() {
101        assert!(is_sourcemap(&PathBuf::from("app.js.map")));
102        assert!(is_sourcemap(&PathBuf::from("APP.JS.MAP")));
103    }
104
105    #[test]
106    fn test_is_sourcemap_css_map() {
107        assert!(is_sourcemap(&PathBuf::from("style.css.map")));
108        assert!(is_sourcemap(&PathBuf::from("STYLE.CSS.MAP")));
109    }
110
111    #[test]
112    fn test_is_sourcemap_not_map() {
113        assert!(!is_sourcemap(&PathBuf::from("app.js")));
114        assert!(!is_sourcemap(&PathBuf::from("data.json")));
115        assert!(!is_sourcemap(&PathBuf::from("other.map")));
116    }
117
118    #[test]
119    fn test_extract_sourcemap_content_basic() {
120        let json = r#"{"version":3,"sourcesContent":["hello\nworld"]}"#;
121        let result = extract_sourcemap_content(json);
122        assert!(result.is_some());
123        let content = result.unwrap();
124        assert!(content.contains("hello"));
125        assert!(content.contains("world"));
126    }
127
128    #[test]
129    fn test_extract_sourcemap_content_mit_license() {
130        let json = r#"{"version":3,"sourcesContent":["Use of this source code is governed by an MIT-style license\nthat can be found in the LICENSE file"]}"#;
131        let result = extract_sourcemap_content(json);
132        assert!(result.is_some());
133        let content = result.unwrap();
134        assert!(content.contains("MIT-style license"));
135        assert!(content.contains("LICENSE file"));
136        assert!(content.contains("\n"));
137    }
138
139    #[test]
140    fn test_extract_sourcemap_content_multiple_entries() {
141        let json = r#"{"version":3,"sourcesContent":["first\nfile","second\nfile"]}"#;
142        let result = extract_sourcemap_content(json);
143        assert!(result.is_some());
144        let content = result.unwrap();
145        assert!(content.contains("first"));
146        assert!(content.contains("second"));
147    }
148
149    #[test]
150    fn test_extract_sourcemap_content_no_sources() {
151        let json = r#"{"version":3,"sources":[]}"#;
152        let result = extract_sourcemap_content(json);
153        assert!(result.is_none());
154    }
155
156    #[test]
157    fn test_extract_sourcemap_content_invalid_json() {
158        let json = r#"not valid json"#;
159        let result = extract_sourcemap_content(json);
160        assert!(result.is_none());
161    }
162
163    #[test]
164    fn test_extract_sourcemap_content_null_entries() {
165        let json = r#"{"version":3,"sourcesContent":[null,"actual\ncontent"]}"#;
166        let result = extract_sourcemap_content(json);
167        assert!(result.is_some());
168        let content = result.unwrap();
169        assert!(content.contains("actual"));
170    }
171
172    #[test]
173    fn test_detection_text_prefers_embedded_sources_for_sourcemaps() {
174        let path = PathBuf::from("bundle.js.map");
175        let raw = r#"{"version":3,"comment":"Copyright 1999 Wrong Corp.","sourcesContent":["/* Copyright 2024 Example Corp. */\n"]}"#;
176
177        let result = detection_text(&path, raw);
178
179        assert_eq!(result.as_ref(), "/* Copyright 2024 Example Corp. */\n");
180    }
181
182    #[test]
183    fn test_replace_verbatim_cr_lf_chars() {
184        // Single-escaped (backslash-n, backslash-r in the string)
185        assert_eq!(replace_verbatim_cr_lf_chars("a\\nb"), "a\nb");
186        assert_eq!(replace_verbatim_cr_lf_chars("a\\rb"), "a\nb");
187        assert_eq!(replace_verbatim_cr_lf_chars("a\\r\\nb"), "a\nb");
188        // Double-escaped (literal backslash-backslash-n in the string)
189        assert_eq!(replace_verbatim_cr_lf_chars("a\\\\nb"), "a\nb");
190        assert_eq!(replace_verbatim_cr_lf_chars("a\\\\rb"), "a\nb");
191        assert_eq!(replace_verbatim_cr_lf_chars("a\\\\r\\\\nb"), "a\nb");
192    }
193
194    #[test]
195    fn test_ar_er_js_map_detection() {
196        let path = PathBuf::from("testdata/license-golden/datadriven/lic2/ar-ER.js.map");
197        if !path.exists() {
198            eprintln!("Skipping test: test file not found");
199            return;
200        }
201
202        let text = std::fs::read_to_string(&path).expect("Failed to read file");
203        eprintln!("Raw text length: {}", text.len());
204
205        let json: serde_json::Value = serde_json::from_str(&text).expect("JSON parse failed");
206        let sources = json
207            .get("sourcesContent")
208            .expect("No sourcesContent")
209            .as_array()
210            .expect("Not array");
211        eprintln!("Sources array length: {}", sources.len());
212
213        if let Some(first) = sources.first().and_then(|v| v.as_str()) {
214            eprintln!("First source length: {}", first.len());
215            eprintln!("First 100 chars: {:?}", &first[..100.min(first.len())]);
216        }
217
218        let result = extract_sourcemap_content(&text);
219        assert!(result.is_some(), "Should extract content from ar-ER.js.map");
220
221        let content = result.unwrap();
222        eprintln!("Extracted content length: {}", content.len());
223        assert!(
224            content.contains("MIT-style license"),
225            "Should contain MIT license text"
226        );
227    }
228}
provenant/utils/sourcemap.rs

provenant/utils/
sourcemap.rs
