provenant/license_detection/rules/
thresholds.rs

1// SPDX-FileCopyrightText: nexB Inc. and others
2// ScanCode is a trademark of nexB Inc.
3// SPDX-FileCopyrightText: Provenant contributors
4// SPDX-License-Identifier: Apache-2.0
5// Derived from ScanCode Toolkit (Apache-2.0); modified. See NOTICE.
6
7//! Compute match thresholds for license detection rules.
8
9/// Minimum match length for token-based matching.
10pub const MIN_MATCH_LENGTH: usize = 4;
11
12/// Minimum match length for high-value (legalese) token matching.
13pub const MIN_MATCH_HIGH_LENGTH: usize = 3;
14
15/// Rules shorter than this are considered "small" (exact match only).
16pub const SMALL_RULE: usize = 15;
17
18/// Rules shorter than this are considered "tiny" (very short, special handling).
19pub const TINY_RULE: usize = 6;
20
21/// Compute thresholds considering the occurrence of all tokens.
22///
23/// This function computes the minimum match thresholds based on the total
24/// length of the rule and the count of high-value (legalese) tokens.
25///
26/// # Arguments
27///
28/// * `minimum_coverage` - Required coverage percentage (0-100), None if not specified
29/// * `length` - Total number of tokens in the rule
30/// * `high_length` - Total count of legalese token occurrences
31///
32/// # Returns
33///
34/// A tuple of (updated_minimum_coverage, min_matched_length, min_high_matched_length)
35pub fn compute_thresholds_occurrences(
36    minimum_coverage: Option<u8>,
37    length: usize,
38    high_length: usize,
39) -> (Option<u8>, usize, usize) {
40    if minimum_coverage == Some(100) {
41        return (minimum_coverage, length, high_length);
42    }
43
44    let (min_matched_length, min_high_matched_length, updated_coverage) = if length < 3 {
45        (length, high_length, Some(100))
46    } else if length < 10 {
47        (length, high_length, Some(80))
48    } else if length < 30 {
49        (length / 2, high_length.min(MIN_MATCH_HIGH_LENGTH), Some(50))
50    } else if length < 200 {
51        (
52            MIN_MATCH_LENGTH,
53            high_length.min(MIN_MATCH_HIGH_LENGTH),
54            minimum_coverage,
55        )
56    } else {
57        (length / 10, high_length / 10, minimum_coverage)
58    };
59
60    (
61        updated_coverage,
62        min_matched_length,
63        min_high_matched_length,
64    )
65}
66
67/// Compute thresholds considering the occurrence of only unique tokens.
68///
69/// This function computes the minimum match thresholds based on the number of
70/// unique tokens in the rule and the count of unique high-value (legalese) tokens.
71///
72/// # Arguments
73///
74/// * `minimum_coverage` - Required coverage percentage (0-100), None if not specified
75/// * `length` - Total number of tokens in the rule
76/// * `length_unique` - Count of unique token IDs in the rule
77/// * `high_length_unique` - Count of unique legalese token IDs
78///
79/// # Returns
80///
81/// A tuple of (min_matched_length_unique, min_high_matched_length_unique)
82pub fn compute_thresholds_unique(
83    minimum_coverage: Option<u8>,
84    length: usize,
85    length_unique: usize,
86    high_length_unique: usize,
87) -> (usize, usize) {
88    if minimum_coverage == Some(100) {
89        return (length_unique, high_length_unique);
90    }
91
92    if length > 200 {
93        (length / 10, high_length_unique / 10)
94    } else if length < 5 {
95        (length_unique, high_length_unique)
96    } else if length < 10 {
97        let min_matched = if length_unique < 2 {
98            length_unique
99        } else {
100            length_unique - 1
101        };
102        (min_matched, high_length_unique)
103    } else if length < 20 {
104        (high_length_unique, high_length_unique)
105    } else {
106        let half = high_length_unique / 2;
107        let high_u = if half > 0 { half } else { high_length_unique };
108        (MIN_MATCH_LENGTH, high_u.min(MIN_MATCH_HIGH_LENGTH))
109    }
110}
111
112#[cfg(test)]
113mod tests {
114    use super::*;
115
116    #[test]
117    fn test_compute_thresholds_occurrences_100_coverage() {
118        let (cov, min_len, min_high_len) = compute_thresholds_occurrences(Some(100), 50, 20);
119        assert_eq!(cov, Some(100));
120        assert_eq!(min_len, 50);
121        assert_eq!(min_high_len, 20);
122    }
123
124    #[test]
125    fn test_compute_thresholds_occurrences_tiny_rule() {
126        let (cov, min_len, min_high_len) = compute_thresholds_occurrences(None, 2, 1);
127        assert_eq!(cov, Some(100));
128        assert_eq!(min_len, 2);
129        assert_eq!(min_high_len, 1);
130    }
131
132    #[test]
133    fn test_compute_thresholds_occurrences_small_rule() {
134        let (cov, min_len, min_high_len) = compute_thresholds_occurrences(None, 8, 3);
135        assert_eq!(cov, Some(80));
136        assert_eq!(min_len, 8);
137        assert_eq!(min_high_len, 3);
138    }
139
140    #[test]
141    fn test_compute_thresholds_occurrences_medium_rule() {
142        let (cov, min_len, min_high_len) = compute_thresholds_occurrences(None, 25, 10);
143        assert_eq!(cov, Some(50));
144        assert_eq!(min_len, 12);
145        assert_eq!(min_high_len, 3);
146    }
147
148    #[test]
149    fn test_compute_thresholds_occurrences_large_rule() {
150        let (cov, min_len, min_high_len) = compute_thresholds_occurrences(None, 100, 40);
151        assert_eq!(cov, None);
152        assert_eq!(min_len, 4);
153        assert_eq!(min_high_len, 3);
154    }
155
156    #[test]
157    fn test_compute_thresholds_occurrences_very_large_rule() {
158        let (cov, min_len, min_high_len) = compute_thresholds_occurrences(None, 500, 200);
159        assert_eq!(cov, None);
160        assert_eq!(min_len, 50);
161        assert_eq!(min_high_len, 20);
162    }
163
164    #[test]
165    fn test_compute_thresholds_unique_100_coverage() {
166        let (min_len, min_high_len) = compute_thresholds_unique(Some(100), 50, 30, 15);
167        assert_eq!(min_len, 30);
168        assert_eq!(min_high_len, 15);
169    }
170
171    #[test]
172    fn test_compute_thresholds_unique_very_large() {
173        let (min_len, min_high_len) = compute_thresholds_unique(None, 500, 300, 150);
174        assert_eq!(min_len, 50);
175        assert_eq!(min_high_len, 15);
176    }
177
178    #[test]
179    fn test_compute_thresholds_unique_tiny() {
180        let (min_len, min_high_len) = compute_thresholds_unique(None, 3, 2, 1);
181        assert_eq!(min_len, 2);
182        assert_eq!(min_high_len, 1);
183    }
184
185    #[test]
186    fn test_compute_thresholds_unique_small() {
187        let (min_len, min_high_len) = compute_thresholds_unique(None, 8, 5, 3);
188        assert_eq!(min_len, 4);
189        assert_eq!(min_high_len, 3);
190    }
191
192    #[test]
193    fn test_compute_thresholds_unique_medium() {
194        let (min_len, min_high_len) = compute_thresholds_unique(None, 15, 10, 5);
195        assert_eq!(min_len, 5);
196        assert_eq!(min_high_len, 5);
197    }
198
199    #[test]
200    fn test_compute_thresholds_unique_large() {
201        let (min_len, min_high_len) = compute_thresholds_unique(None, 100, 40, 20);
202        assert_eq!(min_len, 4);
203        assert_eq!(min_high_len, 3);
204    }
205
206    #[test]
207    fn test_constants() {
208        assert_eq!(MIN_MATCH_LENGTH, 4);
209        assert_eq!(MIN_MATCH_HIGH_LENGTH, 3);
210        assert_eq!(SMALL_RULE, 15);
211        assert_eq!(TINY_RULE, 6);
212    }
213}
214
215#[cfg(test)]
216mod integration_tests {
217    use super::super::super::index::dictionary::{TokenDictionary, TokenId};
218    use super::super::super::models::Rule;
219    use super::*;
220    use crate::license_detection::{TokenMultiset, TokenSet};
221    use std::collections::HashMap;
222
223    /// Helper function to create a rule with mock tokens and compute thresholds.
224    fn create_rule_with_thresholds(
225        text: String,
226        tokens: Vec<u16>,
227        minimum_coverage: Option<u8>,
228        len_legalese: usize,
229    ) -> Rule {
230        let tokens: Vec<TokenId> = tokens.into_iter().map(TokenId::new).collect();
231        let mut rule = Rule {
232            identifier: "test.RULE".to_string(),
233            license_expression: "mit".to_string(),
234            text,
235            tokens: tokens.clone(),
236            rule_kind: crate::license_detection::models::RuleKind::None,
237            is_false_positive: false,
238            is_required_phrase: false,
239            is_from_license: false,
240            relevance: 100,
241            minimum_coverage,
242            has_stored_minimum_coverage: false,
243            is_continuous: false,
244            required_phrase_spans: vec![],
245            stopwords_by_pos: HashMap::new(),
246            referenced_filenames: None,
247            ignorable_urls: None,
248            ignorable_emails: None,
249            ignorable_copyrights: None,
250            ignorable_holders: None,
251            ignorable_authors: None,
252            language: None,
253            notes: None,
254            length_unique: 0,
255            high_length_unique: 0,
256            high_length: 0,
257            min_matched_length: 0,
258            min_high_matched_length: 0,
259            min_matched_length_unique: 0,
260            min_high_matched_length_unique: 0,
261            is_small: false,
262            is_tiny: false,
263            starts_with_license: false,
264            ends_with_license: false,
265            is_deprecated: false,
266            spdx_license_key: None,
267            other_spdx_license_keys: vec![],
268        };
269
270        // Build token sets and multisets
271        let legalese_entries: Vec<(String, u16)> = (0..len_legalese)
272            .map(|i| (format!("legalese-{i}"), i as u16))
273            .collect();
274        let dictionary = TokenDictionary::new_with_legalese_pairs(
275            &legalese_entries
276                .iter()
277                .map(|(token, id)| (token.as_str(), *id))
278                .collect::<Vec<_>>(),
279        );
280        let tids_set = TokenSet::from_token_ids(tokens.iter().copied());
281        let tids_mset = TokenMultiset::from_token_ids(&tokens);
282        let tids_set_high = tids_set.high_subset(&dictionary);
283        let tids_mset_high = tids_mset.high_subset(&dictionary);
284
285        // Compute token counts
286        rule.length_unique = tids_set.len();
287        rule.high_length_unique = tids_set_high.len();
288        rule.high_length = tids_mset_high.total_count();
289
290        // Compute thresholds
291        let (updated_coverage, min_len, min_high_len) =
292            compute_thresholds_occurrences(rule.minimum_coverage, tokens.len(), rule.high_length);
293        rule.minimum_coverage = updated_coverage;
294        rule.min_matched_length = min_len;
295        rule.min_high_matched_length = min_high_len;
296
297        let (min_len_unique, min_high_len_unique) = compute_thresholds_unique(
298            rule.minimum_coverage,
299            tokens.len(),
300            rule.length_unique,
301            rule.high_length_unique,
302        );
303        rule.min_matched_length_unique = min_len_unique;
304        rule.min_high_matched_length_unique = min_high_len_unique;
305
306        // Rule classification
307        rule.is_tiny = tokens.len() < TINY_RULE;
308        rule.is_small = tokens.len() < SMALL_RULE;
309
310        rule
311    }
312
313    #[test]
314    fn test_threshold_computation_with_explicit_coverage() {
315        // Rule with explicit 100% coverage
316        // Note: len_legalese=10 means IDs 0-9 are legalese, so token 10 is NOT legalese
317        let tokens = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 11]; // 11 is not legalese
318        let len_legalese = 10;
319        let rule = create_rule_with_thresholds(
320            "MIT License text".to_string(),
321            tokens.clone(),
322            Some(100),
323            len_legalese,
324        );
325
326        assert_eq!(rule.minimum_coverage, Some(100));
327        assert_eq!(rule.min_matched_length, 10);
328        assert_eq!(rule.min_high_matched_length, 9); // Only 1-9 are legalese
329        assert_eq!(rule.min_matched_length_unique, 10);
330        assert_eq!(rule.min_high_matched_length_unique, 9);
331    }
332
333    #[test]
334    fn test_threshold_computation_full_pipeline_small_rule() {
335        // Small rule with 8 tokens
336        let tokens = vec![1, 2, 3, 4, 6, 7, 12, 15]; // Some legalese (0-9), some not
337        let len_legalese = 10;
338        let rule = create_rule_with_thresholds(
339            "MIT License text here".to_string(),
340            tokens,
341            None,
342            len_legalese,
343        );
344
345        assert!(!rule.is_tiny);
346        assert!(rule.is_small);
347        assert_eq!(rule.length_unique, 8);
348        assert_eq!(rule.high_length_unique, 6); // Tokens 1,2,3,4,6,7 are legalese
349        assert_eq!(rule.high_length, 6);
350        assert_eq!(rule.minimum_coverage, Some(80));
351        assert_eq!(rule.min_matched_length, 8);
352        assert_eq!(rule.min_high_matched_length, 6);
353    }
354
355    #[test]
356    fn test_threshold_computation_full_pipeline_medium_rule() {
357        // Medium rule with 25 tokens
358        let tokens: Vec<u16> = (0..25).collect(); // Many legalese
359        let len_legalese = 10;
360        let rule = create_rule_with_thresholds(
361            "MIT License text here with more words".to_string(),
362            tokens,
363            None,
364            len_legalese,
365        );
366
367        assert!(!rule.is_tiny);
368        assert!(!rule.is_small);
369        assert_eq!(rule.length_unique, 25);
370        assert_eq!(rule.high_length_unique, 10); // Only 0-9 are legalese
371        assert_eq!(rule.high_length, 10);
372        assert_eq!(rule.minimum_coverage, Some(50));
373        assert_eq!(rule.min_matched_length, 12);
374        assert_eq!(rule.min_high_matched_length, 3);
375    }
376
377    #[test]
378    fn test_threshold_computation_full_pipeline_tiny_rule() {
379        // Tiny rule with 3 tokens
380        // Note: In Python, length >= 3 AND length < 10 gives 80% coverage
381        // Only length < 3 gives 100% coverage
382        let tokens = vec![1, 2, 3]; // All legalese
383        let len_legalese = 10;
384        let rule =
385            create_rule_with_thresholds("MIT License".to_string(), tokens, None, len_legalese);
386
387        // TINY_RULE is 6, so a 3-token rule IS tiny and is_small
388        assert!(rule.is_tiny);
389        assert!(rule.is_small);
390        assert_eq!(rule.length_unique, 3);
391        assert_eq!(rule.high_length_unique, 3);
392        assert_eq!(rule.high_length, 3);
393        // For length >= 3 and < 10, coverage is 80%
394        assert_eq!(rule.minimum_coverage, Some(80));
395        assert_eq!(rule.min_matched_length, 3);
396        assert_eq!(rule.min_high_matched_length, 3);
397    }
398
399    #[test]
400    fn test_threshold_computation_unique_token_counts() {
401        // Rule with repeated tokens to test unique counting
402        let tokens = vec![1, 2, 3, 1, 2, 3, 1, 2, 3, 1, 2, 3];
403        let len_legalese = 10;
404        let rule = create_rule_with_thresholds(
405            "MIT License MIT License MIT".to_string(),
406            tokens,
407            None,
408            len_legalese,
409        );
410
411        assert_eq!(rule.length_unique, 3); // Only 3 unique tokens
412        assert_eq!(rule.high_length_unique, 3); // All are legalese
413        assert_eq!(rule.high_length, 12); // But 12 total occurrences
414    }
415
416    #[test]
417    fn test_threshold_computation_no_high_tokens() {
418        // Rule with no legalese tokens (weak rule)
419        let tokens: Vec<u16> = (10..20).collect(); // All IDs >= len_legalese
420        let len_legalese = 10;
421        let rule = create_rule_with_thresholds(
422            "Some text without legal words".to_string(),
423            tokens,
424            None,
425            len_legalese,
426        );
427
428        assert_eq!(rule.high_length_unique, 0);
429        assert_eq!(rule.high_length, 0);
430        assert_eq!(rule.min_high_matched_length, 0);
431    }
432}
provenant/license_detection/rules/thresholds.rs

provenant/license_detection/rules/
thresholds.rs
