cpd_core/
models.rs

1use serde::{Deserialize, Serialize};
2use std::collections::HashMap;
3
4#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
5#[serde(rename_all = "snake_case")]
6pub enum TokenKind {
7    Keyword,
8    Identifier,
9    Literal,
10    Operator,
11    Punctuation,
12    Comment,
13    BlockComment,
14    Whitespace,
15    Ignore,
16    Other,
17}
18
19impl TokenKind {
20    /// Return a stable byte discriminant for use in token hashing.
21    pub fn discriminant(&self) -> u8 {
22        match self {
23            Self::Keyword => 1,
24            Self::Identifier => 2,
25            Self::Literal => 3,
26            Self::Operator => 4,
27            Self::Punctuation => 5,
28            Self::Comment => 6,
29            Self::BlockComment => 7,
30            Self::Whitespace => 8,
31            Self::Ignore => 9,
32            Self::Other => 10,
33        }
34    }
35}
36
37#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
38pub struct Location {
39    pub line: u32,
40    pub column: u32,
41    pub offset: u32,
42}
43
44#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
45pub struct Token {
46    pub kind: TokenKind,
47    pub value: String,
48    pub start: Location,
49    pub end: Location,
50}
51
52#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
53pub struct BlameEntry {
54    pub commit_sha: String,
55    pub author: String,
56    pub timestamp: i64,
57}
58
59#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
60pub struct Fragment {
61    pub source_id: String,
62    pub start: Location,
63    pub end: Location,
64    pub range: [u32; 2],
65    pub blame: Option<BlameEntry>,
66}
67
68#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
69pub struct CpdClone {
70    pub format: String,
71    pub fragment_a: Fragment,
72    pub fragment_b: Fragment,
73    pub token_count: u32,
74}
75
76/// Internal detection unit — no heap allocation per token.
77///
78/// Produced by the tokenizer's detection path at tokenize time.
79/// `Token` is used for display, blame, and reporter output;
80/// `DetectionToken` is used only during the clone detection hot path.
81/// The token's value string is not stored — only its pre-computed hash.
82#[derive(Debug, Clone, PartialEq, Eq)]
83pub struct DetectionToken {
84    /// Pre-computed hash of (kind, value) — detection never re-hashes.
85    pub hash: u64,
86    pub start: Location,
87    pub end: Location,
88    /// Byte range in the source content: `[start_byte, end_byte]`.
89    pub range: [usize; 2],
90}
91
92/// A source file with pre-tokenized tokens, ready for clone detection.
93#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
94pub struct SourceFile {
95    pub id: String,
96    pub format: String,
97    pub tokens: Vec<Token>,
98}
99
100/// Per-format or total statistics row.
101#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
102#[serde(rename_all = "camelCase")]
103pub struct StatRow {
104    pub lines: u64,
105    pub tokens: u64,
106    pub sources: u64,
107    pub clones: u64,
108    pub duplicated_lines: u64,
109    pub duplicated_tokens: u64,
110    pub percentage: f64,
111    pub percentage_tokens: f64,
112    #[serde(default)]
113    pub new_duplicated_lines: u64,
114    #[serde(default)]
115    pub new_clones: u64,
116}
117
118impl Default for StatRow {
119    fn default() -> Self {
120        Self {
121            lines: 0,
122            tokens: 0,
123            sources: 0,
124            clones: 0,
125            duplicated_lines: 0,
126            duplicated_tokens: 0,
127            percentage: 0.0,
128            percentage_tokens: 0.0,
129            new_duplicated_lines: 0,
130            new_clones: 0,
131        }
132    }
133}
134
135/// Aggregated detection statistics.
136#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
137#[serde(rename_all = "camelCase")]
138pub struct Statistics {
139    pub total: StatRow,
140    pub formats: HashMap<String, StatRow>,
141    pub detection_date: String,
142}
143
144#[cfg(test)]
145mod tests {
146    use super::*;
147    use serde_json;
148
149    #[test]
150    fn statistics_default_total_is_zero() {
151        let stats = Statistics {
152            total: StatRow::default(),
153            formats: HashMap::new(),
154            detection_date: "2026-01-01T00:00:00Z".to_string(),
155        };
156        assert_eq!(stats.total.clones, 0);
157    }
158
159    #[test]
160    fn token_serializes_and_deserializes() {
161        let token = Token {
162            kind: TokenKind::Keyword,
163            value: "function".to_string(),
164            start: Location {
165                line: 1,
166                column: 0,
167                offset: 0,
168            },
169            end: Location {
170                line: 1,
171                column: 8,
172                offset: 8,
173            },
174        };
175        let json = serde_json::to_string(&token).unwrap();
176        let back: Token = serde_json::from_str(&json).unwrap();
177        assert_eq!(token, back);
178    }
179
180    #[test]
181    fn cpd_clone_serializes_with_blame() {
182        let loc = Location {
183            line: 1,
184            column: 0,
185            offset: 0,
186        };
187        let blame = BlameEntry {
188            commit_sha: "abc123".to_string(),
189            author: "Alice".to_string(),
190            timestamp: 1700000000,
191        };
192        let frag = Fragment {
193            source_id: "a.js".to_string(),
194            start: loc.clone(),
195            end: loc.clone(),
196            range: [0, 10],
197            blame: Some(blame),
198        };
199        let clone = CpdClone {
200            format: "javascript".to_string(),
201            fragment_a: frag.clone(),
202            fragment_b: frag,
203            token_count: 50,
204        };
205        let json = serde_json::to_string(&clone).unwrap();
206        assert!(json.contains("abc123"));
207        assert!(json.contains("fragment_a"));
208    }
209
210    #[test]
211    fn fragment_blame_none_serializes_as_null() {
212        let loc = Location {
213            line: 1,
214            column: 0,
215            offset: 0,
216        };
217        let frag = Fragment {
218            source_id: "b.js".to_string(),
219            start: loc.clone(),
220            end: loc.clone(),
221            range: [0, 5],
222            blame: None,
223        };
224        let json = serde_json::to_string(&frag).unwrap();
225        assert!(json.contains("\"blame\":null"));
226    }
227}
cpd_core/models.rs

cpd_core/
models.rs
