provenant/utils/
sourcemap.rs

1// SPDX-FileCopyrightText: nexB Inc. and others
2// SPDX-FileCopyrightText: Provenant contributors
3// SPDX-License-Identifier: Apache-2.0
4// Derived from ScanCode Toolkit (Apache-2.0); modified. See NOTICE.
5
6//! Source map file processing for scanner text detection.
7//!
8//! Source map files (.js.map, .css.map) are JSON files containing embedded
9//! source code in a `sourcesContent` array. This module extracts that content
10//! so the scanner can detect licenses and parties from the embedded sources
11//! instead of from the raw source map JSON wrapper.
12
13use std::borrow::Cow;
14use std::path::Path;
15
16/// Check if a file is a source map file based on extension.
17pub fn is_sourcemap(path: &Path) -> bool {
18    path.file_name()
19        .and_then(|name| name.to_str())
20        .map(|name| {
21            let name_lower = name.to_lowercase();
22            name_lower.ends_with(".js.map") || name_lower.ends_with(".css.map")
23        })
24        .unwrap_or(false)
25}
26
27/// Extract source content from a source map JSON file.
28///
29/// Parses the JSON and extracts all entries from `sourcesContent`,
30/// combining them with newlines for license detection.
31///
32/// Returns `Some(combined_text)` if successfully parsed with content.
33/// Returns `None` if JSON parsing fails or no sourcesContent exists.
34pub fn extract_sourcemap_content(json_text: &str) -> Option<String> {
35    let json: serde_json::Value = serde_json::from_str(json_text).ok()?;
36    let sources = json.get("sourcesContent")?.as_array()?;
37
38    let combined: String = sources
39        .iter()
40        .filter_map(|v| v.as_str())
41        .map(replace_verbatim_cr_lf_chars)
42        .collect::<Vec<_>>()
43        .join("\n");
44
45    if combined.is_empty() {
46        None
47    } else {
48        Some(combined)
49    }
50}
51
52/// Return the text scanners should inspect for this file.
53///
54/// Source maps and Jupyter notebooks embed the human-authored text inside a JSON
55/// wrapper; for those we hand scanners the extracted text instead of the raw JSON
56/// so detection sees what a reader would. All other files pass through unchanged.
57pub fn detection_text<'a>(path: &Path, text: &'a str) -> Cow<'a, str> {
58    if is_sourcemap(path) {
59        return extract_sourcemap_content(text)
60            .map(Cow::Owned)
61            .unwrap_or(Cow::Borrowed(text));
62    }
63
64    if crate::utils::notebook::is_notebook(path) {
65        return crate::utils::notebook::extract_notebook_content(text)
66            .map(Cow::Owned)
67            .unwrap_or(Cow::Borrowed(text));
68    }
69
70    Cow::Borrowed(text)
71}
72
73/// Replace verbatim escaped CR/LF characters with actual newlines.
74///
75/// This matches Python's `replace_verbatim_cr_lf_chars()` behavior exactly:
76/// - Double-escaped (e.g., source had literal `\r` that was escaped again):
77///   - `\\r\\n` (backslash-backslash-r-backslash-backslash-n) → newline
78///   - `\\r` (backslash-backslash-r) → newline
79///   - `\\n` (backslash-backslash-n) → newline
80/// - Single-escaped (e.g., JSON-escaped newlines):
81///   - `\r\n` (backslash-r-backslash-n) → newline
82///   - `\r` (backslash-r) → newline
83///   - `\n` (backslash-n) → newline
84fn replace_verbatim_cr_lf_chars(s: &str) -> String {
85    s.replace("\\\\r\\\\n", "\n")
86        .replace("\\r\\n", "\n")
87        .replace("\\\\r", "\n")
88        .replace("\\\\n", "\n")
89        .replace("\\r", "\n")
90        .replace("\\n", "\n")
91}
92
93#[cfg(test)]
94mod tests {
95    use super::*;
96    use std::path::PathBuf;
97
98    #[test]
99    fn test_is_sourcemap_js_map() {
100        assert!(is_sourcemap(&PathBuf::from("app.js.map")));
101        assert!(is_sourcemap(&PathBuf::from("APP.JS.MAP")));
102    }
103
104    #[test]
105    fn test_is_sourcemap_css_map() {
106        assert!(is_sourcemap(&PathBuf::from("style.css.map")));
107        assert!(is_sourcemap(&PathBuf::from("STYLE.CSS.MAP")));
108    }
109
110    #[test]
111    fn test_is_sourcemap_not_map() {
112        assert!(!is_sourcemap(&PathBuf::from("app.js")));
113        assert!(!is_sourcemap(&PathBuf::from("data.json")));
114        assert!(!is_sourcemap(&PathBuf::from("other.map")));
115    }
116
117    #[test]
118    fn test_extract_sourcemap_content_basic() {
119        let json = r#"{"version":3,"sourcesContent":["hello\nworld"]}"#;
120        let result = extract_sourcemap_content(json);
121        assert!(result.is_some());
122        let content = result.unwrap();
123        assert!(content.contains("hello"));
124        assert!(content.contains("world"));
125    }
126
127    #[test]
128    fn test_extract_sourcemap_content_mit_license() {
129        let json = r#"{"version":3,"sourcesContent":["Use of this source code is governed by an MIT-style license\nthat can be found in the LICENSE file"]}"#;
130        let result = extract_sourcemap_content(json);
131        assert!(result.is_some());
132        let content = result.unwrap();
133        assert!(content.contains("MIT-style license"));
134        assert!(content.contains("LICENSE file"));
135        assert!(content.contains("\n"));
136    }
137
138    #[test]
139    fn test_extract_sourcemap_content_multiple_entries() {
140        let json = r#"{"version":3,"sourcesContent":["first\nfile","second\nfile"]}"#;
141        let result = extract_sourcemap_content(json);
142        assert!(result.is_some());
143        let content = result.unwrap();
144        assert!(content.contains("first"));
145        assert!(content.contains("second"));
146    }
147
148    #[test]
149    fn test_extract_sourcemap_content_no_sources() {
150        let json = r#"{"version":3,"sources":[]}"#;
151        let result = extract_sourcemap_content(json);
152        assert!(result.is_none());
153    }
154
155    #[test]
156    fn test_extract_sourcemap_content_invalid_json() {
157        let json = r#"not valid json"#;
158        let result = extract_sourcemap_content(json);
159        assert!(result.is_none());
160    }
161
162    #[test]
163    fn test_extract_sourcemap_content_null_entries() {
164        let json = r#"{"version":3,"sourcesContent":[null,"actual\ncontent"]}"#;
165        let result = extract_sourcemap_content(json);
166        assert!(result.is_some());
167        let content = result.unwrap();
168        assert!(content.contains("actual"));
169    }
170
171    #[test]
172    fn test_detection_text_prefers_embedded_sources_for_sourcemaps() {
173        let path = PathBuf::from("bundle.js.map");
174        let raw = r#"{"version":3,"comment":"Copyright 1999 Wrong Corp.","sourcesContent":["/* Copyright 2024 Example Corp. */\n"]}"#;
175
176        let result = detection_text(&path, raw);
177
178        assert_eq!(result.as_ref(), "/* Copyright 2024 Example Corp. */\n");
179    }
180
181    #[test]
182    fn test_replace_verbatim_cr_lf_chars() {
183        // Single-escaped (backslash-n, backslash-r in the string)
184        assert_eq!(replace_verbatim_cr_lf_chars("a\\nb"), "a\nb");
185        assert_eq!(replace_verbatim_cr_lf_chars("a\\rb"), "a\nb");
186        assert_eq!(replace_verbatim_cr_lf_chars("a\\r\\nb"), "a\nb");
187        // Double-escaped (literal backslash-backslash-n in the string)
188        assert_eq!(replace_verbatim_cr_lf_chars("a\\\\nb"), "a\nb");
189        assert_eq!(replace_verbatim_cr_lf_chars("a\\\\rb"), "a\nb");
190        assert_eq!(replace_verbatim_cr_lf_chars("a\\\\r\\\\nb"), "a\nb");
191    }
192
193    #[test]
194    fn test_ar_er_js_map_detection() {
195        let path = PathBuf::from("testdata/license-golden/datadriven/lic2/ar-ER.js.map");
196        if !path.exists() {
197            eprintln!("Skipping test: test file not found");
198            return;
199        }
200
201        let text = std::fs::read_to_string(&path).expect("Failed to read file");
202        eprintln!("Raw text length: {}", text.len());
203
204        let json: serde_json::Value = serde_json::from_str(&text).expect("JSON parse failed");
205        let sources = json
206            .get("sourcesContent")
207            .expect("No sourcesContent")
208            .as_array()
209            .expect("Not array");
210        eprintln!("Sources array length: {}", sources.len());
211
212        if let Some(first) = sources.first().and_then(|v| v.as_str()) {
213            eprintln!("First source length: {}", first.len());
214            eprintln!("First 100 chars: {:?}", &first[..100.min(first.len())]);
215        }
216
217        let result = extract_sourcemap_content(&text);
218        assert!(result.is_some(), "Should extract content from ar-ER.js.map");
219
220        let content = result.unwrap();
221        eprintln!("Extracted content length: {}", content.len());
222        assert!(
223            content.contains("MIT-style license"),
224            "Should contain MIT license text"
225        );
226    }
227}
provenant/utils/sourcemap.rs

provenant/utils/
sourcemap.rs
