provenant-cli 0.0.33

Rust-based ScanCode-compatible scanner for licenses, package metadata, SBOMs, and provenance data.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126

// SPDX-FileCopyrightText: Provenant contributors
// SPDX-License-Identifier: Apache-2.0

use std::collections::HashMap;
use std::ops::Deref;

use rkyv::Archive;

use crate::license_detection::index::dictionary::{TokenDictionary, TokenId, TokenKind};

/// A multiset of token IDs stored as token -> occurrence count.
#[derive(Clone, Debug, PartialEq, Eq, Default, Archive, rkyv::Serialize, rkyv::Deserialize)]
pub struct TokenMultiset(HashMap<TokenId, usize>);

impl TokenMultiset {
    /// Create a TokenMultiset from a sequence of token IDs.
    pub fn from_token_ids(token_ids: &[TokenId]) -> Self {
        let mut counts = HashMap::new();

        for &tid in token_ids {
            *counts.entry(tid).or_insert(0) += 1;
        }

        Self(counts)
    }

    /// Total number of token occurrences in the multiset.
    pub fn total_count(&self) -> usize {
        self.0.values().sum()
    }

    /// Get a subset containing only high-value (legalese) tokens.
    pub fn high_subset(&self, dictionary: &TokenDictionary) -> Self {
        self.0
            .iter()
            .filter(|(tid, _)| dictionary.token_kind(**tid) == TokenKind::Legalese)
            .map(|(&tid, &count)| (tid, count))
            .collect()
    }

    /// Materialize the multiset intersection with another TokenMultiset.
    pub fn intersection(&self, other: &Self) -> Self {
        let (smaller, larger) = if self.0.len() < other.0.len() {
            (&self.0, &other.0)
        } else {
            (&other.0, &self.0)
        };

        smaller
            .iter()
            .filter_map(|(&tid, &count)| {
                larger
                    .get(&tid)
                    .map(|&other_count| (tid, count.min(other_count)))
            })
            .collect()
    }
}

impl Deref for TokenMultiset {
    type Target = HashMap<TokenId, usize>;

    fn deref(&self) -> &Self::Target {
        &self.0
    }
}

impl FromIterator<(TokenId, usize)> for TokenMultiset {
    fn from_iter<T: IntoIterator<Item = (TokenId, usize)>>(iter: T) -> Self {
        Self(iter.into_iter().collect())
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::license_detection::index::dictionary::{TokenDictionary, tid};

    #[test]
    fn test_from_token_ids() {
        let token_ids = vec![tid(1), tid(2), tid(3), tid(2), tid(4), tid(1), tid(1)];
        let multiset = TokenMultiset::from_token_ids(&token_ids);

        assert_eq!(multiset.get(&tid(1)), Some(&3));
        assert_eq!(multiset.get(&tid(2)), Some(&2));
        assert_eq!(multiset.get(&tid(3)), Some(&1));
        assert_eq!(multiset.get(&tid(4)), Some(&1));
    }

    #[test]
    fn test_total_count() {
        let token_ids = vec![tid(1), tid(2), tid(3), tid(2), tid(1), tid(1)];
        let multiset = TokenMultiset::from_token_ids(&token_ids);

        assert_eq!(multiset.total_count(), 6);
    }

    #[test]
    fn test_high_subset() {
        let token_ids = vec![tid(1), tid(1), tid(2), tid(5), tid(10)];
        let multiset = TokenMultiset::from_token_ids(&token_ids);
        let dict = TokenDictionary::new_with_legalese_pairs(&[("one", 1), ("two", 2)]);

        let high_multiset = multiset.high_subset(&dict);

        assert_eq!(high_multiset.len(), 2);
        assert_eq!(high_multiset.get(&tid(1)), Some(&2));
        assert_eq!(high_multiset.get(&tid(2)), Some(&1));
        assert!(!high_multiset.contains_key(&tid(5)));
        assert!(!high_multiset.contains_key(&tid(10)));
    }

    #[test]
    fn test_intersection() {
        let left = TokenMultiset::from_token_ids(&[tid(1), tid(1), tid(2), tid(3)]);
        let right = TokenMultiset::from_token_ids(&[tid(1), tid(2), tid(2), tid(4)]);

        let intersection = left.intersection(&right);

        assert_eq!(intersection.get(&tid(1)), Some(&1));
        assert_eq!(intersection.get(&tid(2)), Some(&1));
        assert!(!intersection.contains_key(&tid(3)));
        assert!(!intersection.contains_key(&tid(4)));
        assert_eq!(intersection.total_count(), 2);
    }
}