brokk_bifrost_python/
clones.rs

1//! Python's clone-detection token and AST-signature normalization.
2//!
3//! `analyzer/python/clones.rs` in `brokk-bifrost-analysis` keeps the entry
4//! point: it reads the declaration's source through the analyzer and assembles
5//! the analysis-owned `CloneCandidateData`. Everything that knows what a Python
6//! token *is* lives here.
7
8use crate::declarations::parse_python_tree;
9use brokk_bifrost_core::clones::{
10    CloneNormalizationRules, clone_ast_signature, clone_ast_signature_from_root,
11    normalized_clone_tokens, normalized_clone_tokens_from_root,
12};
13#[cfg(test)]
14use brokk_bifrost_core::clones::{
15    normalize_clone_ast_label as normalize_python_clone_ast_label,
16    normalize_clone_leaf_token as normalize_python_clone_leaf_token,
17};
18#[cfg(any(test, feature = "test-support"))]
19use std::cell::Cell;
20
21#[cfg(any(test, feature = "test-support"))]
22thread_local! {
23    static CLONE_PARSE_CALLS: Cell<usize> = const { Cell::new(0) };
24}
25
26#[derive(Debug, Clone, PartialEq, Eq)]
27pub struct PythonClonePreparation {
28    pub normalized_tokens: Vec<String>,
29    pub ast_signature: String,
30}
31
32#[cfg(any(test, feature = "test-support"))]
33pub fn reset_python_clone_parse_count_for_test() {
34    CLONE_PARSE_CALLS.set(0);
35}
36
37#[cfg(any(test, feature = "test-support"))]
38pub fn python_clone_parse_count_for_test() -> usize {
39    CLONE_PARSE_CALLS.get()
40}
41
42fn parse_clone_tree(source: &str) -> Option<tree_sitter::Tree> {
43    #[cfg(any(test, feature = "test-support"))]
44    CLONE_PARSE_CALLS.set(CLONE_PARSE_CALLS.get() + 1);
45    parse_python_tree(source)
46}
47
48const PYTHON_CLONE_AST_IDENTIFIER_TYPES: &[&str] = &["identifier", "keyword_identifier"];
49const PYTHON_CLONE_AST_STRING_TYPES: &[&str] = &["string", "string_content", "interpolation"];
50const PYTHON_CLONE_AST_NUMBER_TYPES: &[&str] = &["integer", "float"];
51const PYTHON_CLONE_NORMALIZATION_RULES: CloneNormalizationRules = CloneNormalizationRules {
52    identifier_kinds: PYTHON_CLONE_AST_IDENTIFIER_TYPES,
53    string_kinds: PYTHON_CLONE_AST_STRING_TYPES,
54    number_kinds: PYTHON_CLONE_AST_NUMBER_TYPES,
55    boolean_kinds: &["true", "false"],
56    boolean_values: &["True", "False"],
57    null_kinds: &[],
58    null_values: &[],
59    comment_kinds: &["comment"],
60    ignored_ast_kinds: &[],
61};
62
63pub fn normalized_clone_tokens_python(source: &str) -> Vec<String> {
64    normalized_clone_tokens(
65        parse_clone_tree(source),
66        source,
67        PYTHON_CLONE_NORMALIZATION_RULES,
68    )
69}
70
71pub fn build_python_clone_ast_signature(source: &str) -> String {
72    clone_ast_signature(
73        parse_clone_tree(source),
74        source,
75        PYTHON_CLONE_NORMALIZATION_RULES,
76    )
77}
78
79/// Parse and prepare one Python clone candidate, stopping before AST-label work
80/// when the normalized token threshold cannot be met.
81pub fn prepare_python_clone(
82    source: &str,
83    min_normalized_tokens: usize,
84) -> Option<PythonClonePreparation> {
85    let Some(tree) = parse_clone_tree(source) else {
86        return (min_normalized_tokens == 0).then(|| PythonClonePreparation {
87            normalized_tokens: Vec::new(),
88            ast_signature: String::new(),
89        });
90    };
91    let normalized_tokens = normalized_clone_tokens_from_root(
92        tree.root_node(),
93        source,
94        PYTHON_CLONE_NORMALIZATION_RULES,
95    );
96    if normalized_tokens.len() < min_normalized_tokens {
97        return None;
98    }
99
100    Some(PythonClonePreparation {
101        normalized_tokens,
102        ast_signature: clone_ast_signature_from_root(
103            tree.root_node(),
104            source,
105            PYTHON_CLONE_NORMALIZATION_RULES,
106        ),
107    })
108}
109
110#[cfg(test)]
111mod tests {
112    use super::*;
113    use brokk_bifrost_test_macros::hardwon;
114
115    #[test]
116    #[hardwon("#3539: clone preparation changed output or parsed each candidate more than once")]
117    fn combined_preparation_matches_the_existing_outputs_with_one_parse() {
118        let sources = [
119            r#"
120def compare(left, right):
121    # Keep comments in the AST signature, but not the token stream.
122    message = f"same: {left}"
123    ratio = 1.5 + 0x2A
124    return left == right and True
125"#,
126            "def broken(:\n    return False + 1\n",
127        ];
128
129        for source in sources {
130            let tree = parse_python_tree(source).unwrap();
131            if source.contains("broken") {
132                assert!(tree.root_node().has_error());
133            }
134            let mut expected_tokens = Vec::new();
135            brokk_bifrost_core::test_support::walk_tree_preorder_reference(
136                tree.root_node(),
137                |node| {
138                    if node.named_child_count() == 0 {
139                        let token = normalize_python_clone_leaf_token(
140                            node,
141                            source,
142                            PYTHON_CLONE_NORMALIZATION_RULES,
143                        );
144                        if !token.is_empty() {
145                            expected_tokens.push(token);
146                        }
147                    }
148                },
149            );
150            let mut expected_labels = Vec::new();
151            brokk_bifrost_core::test_support::walk_tree_preorder_reference(
152                tree.root_node(),
153                |node| {
154                    expected_labels.push(normalize_python_clone_ast_label(
155                        node,
156                        source,
157                        PYTHON_CLONE_NORMALIZATION_RULES,
158                    ));
159                },
160            );
161
162            reset_python_clone_parse_count_for_test();
163            let preparation = prepare_python_clone(source, 0).unwrap();
164
165            assert_eq!(preparation.normalized_tokens, expected_tokens);
166            assert_eq!(preparation.ast_signature, expected_labels.join("|"));
167            assert_eq!(python_clone_parse_count_for_test(), 1);
168            if source.contains("compare") {
169                for expected in ["STR", "NUM", "BOOL", "OP:="] {
170                    assert!(
171                        preparation
172                            .normalized_tokens
173                            .iter()
174                            .any(|token| token == expected),
175                        "missing normalized token {expected}"
176                    );
177                }
178                assert!(
179                    preparation
180                        .normalized_tokens
181                        .iter()
182                        .all(|token| token != "T:comment")
183                );
184                assert!(
185                    preparation
186                        .ast_signature
187                        .split('|')
188                        .any(|label| label == "N:comment")
189                );
190            }
191        }
192    }
193
194    #[test]
195    #[hardwon("#3539: rejected clone candidates repeated source parsing")]
196    fn rejected_candidate_still_uses_one_parse() {
197        let token_count = prepare_python_clone("value = 1", 0)
198            .unwrap()
199            .normalized_tokens
200            .len();
201        reset_python_clone_parse_count_for_test();
202
203        assert!(prepare_python_clone("value = 1", token_count).is_some());
204        assert_eq!(python_clone_parse_count_for_test(), 1);
205        reset_python_clone_parse_count_for_test();
206
207        assert!(prepare_python_clone("value = 1", token_count + 1).is_none());
208        assert_eq!(python_clone_parse_count_for_test(), 1);
209    }
210
211    #[test]
212    #[hardwon("#3539: clone preparation recursively overflowed on deeply nested syntax")]
213    fn deeply_nested_source_is_walked_without_recursion() {
214        let depth = 2_000;
215        let source = format!("value = {}0{}\n", "(".repeat(depth), ")".repeat(depth));
216
217        let preparation = prepare_python_clone(&source, 0).unwrap();
218        brokk_bifrost_core::test_support::assert_deep_clone_preparation(
219            &preparation.normalized_tokens,
220            &preparation.ast_signature,
221            depth,
222        );
223    }
224}
brokk_bifrost_python/clones.rs

brokk_bifrost_python/
clones.rs
