provenant/license_detection/models/
rule.rs

1// SPDX-FileCopyrightText: nexB Inc. and others
2// SPDX-FileCopyrightText: Provenant contributors
3// SPDX-License-Identifier: Apache-2.0
4// Derived from ScanCode Toolkit (Apache-2.0); modified. See NOTICE.
5
6//! Rule metadata loaded from .LICENSE and .RULE files.
7
8use std::collections::HashMap;
9use std::ops::Range;
10
11use rkyv::Archive;
12use serde::{Deserialize, Serialize};
13
14use crate::license_detection::index::dictionary::TokenId;
15
16const SCANCODE_LICENSE_URL_BASE: &str =
17    "https://github.com/aboutcode-org/scancode-toolkit/tree/develop/src/licensedcode/data/licenses";
18const SCANCODE_RULE_URL_BASE: &str =
19    "https://github.com/aboutcode-org/scancode-toolkit/tree/develop/src/licensedcode/data/rules";
20
21mod range_serde {
22    use serde::{Deserialize, Deserializer, Serialize, Serializer};
23    use std::ops::Range;
24
25    pub fn serialize<S>(ranges: &[Range<usize>], serializer: S) -> Result<S::Ok, S::Error>
26    where
27        S: Serializer,
28    {
29        let tuples: Vec<(usize, usize)> = ranges.iter().map(|r| (r.start, r.end)).collect();
30        tuples.serialize(serializer)
31    }
32
33    pub fn deserialize<'de, D>(deserializer: D) -> Result<Vec<Range<usize>>, D::Error>
34    where
35        D: Deserializer<'de>,
36    {
37        let tuples: Vec<(usize, usize)> = Vec::deserialize(deserializer)?;
38        Ok(tuples
39            .into_iter()
40            .map(|(start, end)| Range { start, end })
41            .collect())
42    }
43}
44
45mod stopwords_serde {
46    use serde::{Deserialize, Deserializer, Serialize, Serializer};
47    use std::collections::HashMap;
48
49    pub fn serialize<S>(
50        map: &HashMap<Option<usize>, usize>,
51        serializer: S,
52    ) -> Result<S::Ok, S::Error>
53    where
54        S: Serializer,
55    {
56        let mut entries: Vec<(Option<usize>, usize)> = map.iter().map(|(k, v)| (*k, *v)).collect();
57        entries.sort_by_key(|(k, _)| *k);
58        entries.serialize(serializer)
59    }
60
61    pub fn deserialize<'de, D>(deserializer: D) -> Result<HashMap<Option<usize>, usize>, D::Error>
62    where
63        D: Deserializer<'de>,
64    {
65        let entries: Vec<(Option<usize>, usize)> = Vec::deserialize(deserializer)?;
66        Ok(entries.into_iter().collect())
67    }
68}
69
70#[derive(
71    Debug,
72    Clone,
73    Copy,
74    PartialEq,
75    Eq,
76    Hash,
77    PartialOrd,
78    Ord,
79    Default,
80    Serialize,
81    Deserialize,
82    Archive,
83    rkyv::Serialize,
84    rkyv::Deserialize,
85)]
86#[rkyv(derive(Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord))]
87pub enum RuleKind {
88    #[default]
89    None,
90    Text,
91    Notice,
92    Reference,
93    Tag,
94    Intro,
95    Clue,
96}
97
98impl RuleKind {
99    pub fn from_rule_flags(
100        is_license_text: bool,
101        is_license_notice: bool,
102        is_license_reference: bool,
103        is_license_tag: bool,
104        is_license_intro: bool,
105        is_license_clue: bool,
106    ) -> Result<Self, &'static str> {
107        let mut active = None;
108
109        for (enabled, kind) in [
110            (is_license_text, Self::Text),
111            (is_license_notice, Self::Notice),
112            (is_license_reference, Self::Reference),
113            (is_license_tag, Self::Tag),
114            (is_license_intro, Self::Intro),
115            (is_license_clue, Self::Clue),
116        ] {
117            if !enabled {
118                continue;
119            }
120
121            if active.replace(kind).is_some() {
122                return Err("rule has multiple rule kinds set");
123            }
124        }
125
126        Ok(active.unwrap_or(Self::None))
127    }
128
129    pub const fn is_license_text(self) -> bool {
130        matches!(self, Self::Text)
131    }
132
133    pub const fn is_license_notice(self) -> bool {
134        matches!(self, Self::Notice)
135    }
136
137    pub const fn is_license_reference(self) -> bool {
138        matches!(self, Self::Reference)
139    }
140
141    pub const fn is_license_tag(self) -> bool {
142        matches!(self, Self::Tag)
143    }
144
145    pub const fn is_license_intro(self) -> bool {
146        matches!(self, Self::Intro)
147    }
148
149    pub const fn is_license_clue(self) -> bool {
150        matches!(self, Self::Clue)
151    }
152}
153
154/// Rule metadata loaded from .LICENSE and .RULE files.
155#[derive(
156    Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Archive, rkyv::Serialize, rkyv::Deserialize,
157)]
158pub struct Rule {
159    /// Unique identifier for this rule (e.g., "mit.LICENSE", "gpl-2.0_12.RULE")
160    /// Used for sorting to match Python's attr.s field order.
161    /// This is the primary sort key after rid (which is None at sort time in Python).
162    pub identifier: String,
163
164    /// License expression string using SPDX syntax and ScanCode license keys
165    pub license_expression: String,
166
167    /// Pattern text to match
168    pub text: String,
169
170    /// Token IDs for the text (assigned during indexing)
171    #[serde(
172        serialize_with = "serialize_token_ids",
173        deserialize_with = "deserialize_token_ids"
174    )]
175    pub tokens: Vec<TokenId>,
176
177    /// Classification of this rule.
178    pub rule_kind: RuleKind,
179
180    /// True if exact matches to this rule are false positives
181    pub is_false_positive: bool,
182
183    /// True if this rule text is a required phrase.
184    /// A required phrase is an essential section of the rule text which must be
185    /// present in the case of partial matches.
186    pub is_required_phrase: bool,
187
188    /// True if this rule was created from a license file (not a .RULE file)
189    pub is_from_license: bool,
190
191    /// Relevance score 0-100 (100 is most relevant)
192    pub relevance: u8,
193
194    /// Minimum match coverage percentage (0-100) if specified
195    pub minimum_coverage: Option<u8>,
196
197    /// True if minimum_coverage was explicitly stored in source frontmatter
198    pub has_stored_minimum_coverage: bool,
199
200    /// Tokens must appear in order if true
201    pub is_continuous: bool,
202
203    /// Token position spans for required phrases parsed from {{...}} markers.
204    /// Each span represents positions in the rule text that MUST be matched.
205    #[serde(with = "range_serde", default)]
206    pub required_phrase_spans: Vec<Range<usize>>,
207
208    /// Mapping from token position to count of stopwords at that position.
209    /// Used for required phrase validation.
210    #[serde(with = "stopwords_serde", default)]
211    pub stopwords_by_pos: HashMap<Option<usize>, usize>,
212
213    /// Filenames where this rule should be considered
214    pub referenced_filenames: Option<Vec<String>>,
215
216    /// URLs that should be ignored when found in this rule text
217    pub ignorable_urls: Option<Vec<String>>,
218
219    /// Emails that should be ignored when found in this rule text
220    pub ignorable_emails: Option<Vec<String>>,
221
222    /// Copyrights that should be ignored when found in this rule text
223    pub ignorable_copyrights: Option<Vec<String>>,
224
225    /// Holder names that should be ignored when found in this rule text
226    pub ignorable_holders: Option<Vec<String>>,
227
228    /// Author names that should be ignored when found in this rule text
229    pub ignorable_authors: Option<Vec<String>>,
230
231    /// Programming language for the rule if specified
232    pub language: Option<String>,
233
234    /// Free text notes
235    pub notes: Option<String>,
236
237    /// Count of unique token IDs in the rule (computed during indexing)
238    pub length_unique: usize,
239
240    /// Count of unique legalese token IDs (tokens with ID < len_legalese)
241    pub high_length_unique: usize,
242
243    /// Total count of legalese token occurrences (with duplicates)
244    pub high_length: usize,
245
246    /// Minimum matched length threshold (occurrences-based)
247    pub min_matched_length: usize,
248
249    /// Minimum high-value token matched length threshold (occurrences-based)
250    pub min_high_matched_length: usize,
251
252    /// Minimum matched length threshold (unique tokens)
253    pub min_matched_length_unique: usize,
254
255    /// Minimum high-value token matched length threshold (unique tokens)
256    pub min_high_matched_length_unique: usize,
257
258    /// True if rule length < SMALL_RULE (15 tokens)
259    pub is_small: bool,
260
261    /// True if rule length < TINY_RULE (6 tokens)
262    pub is_tiny: bool,
263
264    /// True if the rule's first token is "license", "licence", or "licensed"
265    pub starts_with_license: bool,
266
267    /// True if the rule's last token is "license", "licence", or "licensed"
268    pub ends_with_license: bool,
269
270    /// Whether this rule is deprecated
271    pub is_deprecated: bool,
272
273    /// SPDX license identifier if available
274    pub spdx_license_key: Option<String>,
275
276    /// Alternative SPDX license identifiers (aliases)
277    pub other_spdx_license_keys: Vec<String>,
278}
279
280fn serialize_token_ids<S>(token_ids: &[TokenId], serializer: S) -> Result<S::Ok, S::Error>
281where
282    S: serde::Serializer,
283{
284    let raw_ids: Vec<u16> = token_ids.iter().map(|id| id.raw()).collect();
285    <Vec<u16> as serde::Serialize>::serialize(&raw_ids, serializer)
286}
287
288fn deserialize_token_ids<'de, D>(deserializer: D) -> Result<Vec<TokenId>, D::Error>
289where
290    D: serde::Deserializer<'de>,
291{
292    let raw_ids: Vec<u16> = Vec::deserialize(deserializer)?;
293    Ok(raw_ids.into_iter().map(TokenId::new).collect())
294}
295
296impl PartialOrd for Rule {
297    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
298        Some(self.cmp(other))
299    }
300}
301
302impl Ord for Rule {
303    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
304        self.identifier.cmp(&other.identifier)
305    }
306}
307
308impl Rule {
309    pub fn rule_url(&self) -> Option<String> {
310        if self.is_from_license {
311            return (!self.license_expression.is_empty()).then(|| {
312                format!(
313                    "{SCANCODE_LICENSE_URL_BASE}/{}.LICENSE",
314                    self.license_expression
315                )
316            });
317        }
318
319        (!self.identifier.is_empty())
320            .then(|| format!("{SCANCODE_RULE_URL_BASE}/{}", self.identifier))
321    }
322
323    pub const fn kind(&self) -> RuleKind {
324        self.rule_kind
325    }
326
327    pub const fn is_license_text(&self) -> bool {
328        self.rule_kind.is_license_text()
329    }
330
331    /// Returns true if this rule is a license notice pattern.
332    ///
333    /// Note: This method is kept for API completeness and potential future use.
334    /// License matches cannot have `is_license_notice` - only rules can.
335    #[allow(dead_code)]
336    pub const fn is_license_notice(&self) -> bool {
337        self.rule_kind.is_license_notice()
338    }
339
340    pub const fn is_license_reference(&self) -> bool {
341        self.rule_kind.is_license_reference()
342    }
343
344    pub const fn is_license_tag(&self) -> bool {
345        self.rule_kind.is_license_tag()
346    }
347
348    /// Returns true if this rule is a license introduction pattern.
349    ///
350    /// Note: This method is kept for API completeness and potential future use.
351    #[allow(dead_code)]
352    pub const fn is_license_intro(&self) -> bool {
353        self.rule_kind.is_license_intro()
354    }
355
356    pub const fn is_license_clue(&self) -> bool {
357        self.rule_kind.is_license_clue()
358    }
359}
provenant/license_detection/models/rule.rs

provenant/license_detection/models/
rule.rs
