brokk_bifrost_python/
clones.rs

1//! Python's clone-detection token and AST-signature normalization.
2//!
3//! `analyzer/python/clones.rs` in `brokk-bifrost-analysis` keeps the entry
4//! point: it reads the declaration's source through the analyzer and assembles
5//! the analysis-owned `CloneCandidateData`. Everything that knows what a Python
6//! token *is* lives here.
7
8use crate::declarations::parse_python_tree;
9use brokk_bifrost_core::clones::{
10    CloneNormalizationRules, clone_ast_signature_from_root, normalized_clone_tokens_from_root,
11};
12#[cfg(test)]
13use brokk_bifrost_core::clones::{
14    normalize_clone_ast_label as normalize_python_clone_ast_label,
15    normalize_clone_leaf_token as normalize_python_clone_leaf_token,
16};
17#[cfg(any(test, feature = "test-support"))]
18use std::cell::Cell;
19
20#[cfg(any(test, feature = "test-support"))]
21thread_local! {
22    static CLONE_PARSE_CALLS: Cell<usize> = const { Cell::new(0) };
23}
24
25#[derive(Debug, Clone, PartialEq, Eq)]
26pub struct PythonClonePreparation {
27    pub normalized_tokens: Vec<String>,
28    pub ast_signature: String,
29}
30
31#[cfg(any(test, feature = "test-support"))]
32pub fn reset_python_clone_parse_count_for_test() {
33    CLONE_PARSE_CALLS.set(0);
34}
35
36#[cfg(any(test, feature = "test-support"))]
37pub fn python_clone_parse_count_for_test() -> usize {
38    CLONE_PARSE_CALLS.get()
39}
40
41fn parse_clone_tree(source: &str) -> Option<tree_sitter::Tree> {
42    #[cfg(any(test, feature = "test-support"))]
43    CLONE_PARSE_CALLS.set(CLONE_PARSE_CALLS.get() + 1);
44    parse_python_tree(source)
45}
46
47const PYTHON_CLONE_AST_IDENTIFIER_TYPES: &[&str] = &["identifier", "keyword_identifier"];
48const PYTHON_CLONE_AST_STRING_TYPES: &[&str] = &["string", "string_content", "interpolation"];
49const PYTHON_CLONE_AST_NUMBER_TYPES: &[&str] = &["integer", "float"];
50const PYTHON_CLONE_NORMALIZATION_RULES: CloneNormalizationRules = CloneNormalizationRules {
51    identifier_kinds: PYTHON_CLONE_AST_IDENTIFIER_TYPES,
52    string_kinds: PYTHON_CLONE_AST_STRING_TYPES,
53    number_kinds: PYTHON_CLONE_AST_NUMBER_TYPES,
54    boolean_kinds: &["true", "false"],
55    boolean_values: &["True", "False"],
56    null_kinds: &[],
57    null_values: &[],
58    comment_kinds: &["comment"],
59    ignored_ast_kinds: &[],
60};
61
62/// Parse and prepare one Python clone candidate, stopping before AST-label work
63/// when the normalized token threshold cannot be met.
64pub fn prepare_python_clone(
65    source: &str,
66    min_normalized_tokens: usize,
67) -> Option<PythonClonePreparation> {
68    let Some(tree) = parse_clone_tree(source) else {
69        return (min_normalized_tokens == 0).then(|| PythonClonePreparation {
70            normalized_tokens: Vec::new(),
71            ast_signature: String::new(),
72        });
73    };
74    let normalized_tokens = normalized_clone_tokens_from_root(
75        tree.root_node(),
76        source,
77        PYTHON_CLONE_NORMALIZATION_RULES,
78    );
79    if normalized_tokens.len() < min_normalized_tokens {
80        return None;
81    }
82
83    Some(PythonClonePreparation {
84        normalized_tokens,
85        ast_signature: clone_ast_signature_from_root(
86            tree.root_node(),
87            source,
88            PYTHON_CLONE_NORMALIZATION_RULES,
89        ),
90    })
91}
92
93#[cfg(test)]
94mod tests {
95    use super::*;
96    use brokk_bifrost_test_macros::hardwon;
97
98    #[test]
99    #[hardwon("#3539: clone preparation changed output or parsed each candidate more than once")]
100    fn combined_preparation_matches_the_existing_outputs_with_one_parse() {
101        let sources = [
102            r#"
103def compare(left, right):
104    # Keep comments in the AST signature, but not the token stream.
105    message = f"same: {left}"
106    ratio = 1.5 + 0x2A
107    return left == right and True
108"#,
109            "def broken(:\n    return False + 1\n",
110        ];
111
112        for source in sources {
113            let tree = parse_python_tree(source).unwrap();
114            if source.contains("broken") {
115                assert!(tree.root_node().has_error());
116            }
117            let mut expected_tokens = Vec::new();
118            brokk_bifrost_core::test_support::walk_tree_preorder_reference(
119                tree.root_node(),
120                |node| {
121                    if node.named_child_count() == 0 {
122                        let token = normalize_python_clone_leaf_token(
123                            node,
124                            source,
125                            PYTHON_CLONE_NORMALIZATION_RULES,
126                        );
127                        if !token.is_empty() {
128                            expected_tokens.push(token);
129                        }
130                    }
131                },
132            );
133            let mut expected_labels = Vec::new();
134            brokk_bifrost_core::test_support::walk_tree_preorder_reference(
135                tree.root_node(),
136                |node| {
137                    expected_labels.push(normalize_python_clone_ast_label(
138                        node,
139                        source,
140                        PYTHON_CLONE_NORMALIZATION_RULES,
141                    ));
142                },
143            );
144
145            reset_python_clone_parse_count_for_test();
146            let preparation = prepare_python_clone(source, 0).unwrap();
147
148            assert_eq!(preparation.normalized_tokens, expected_tokens);
149            assert_eq!(preparation.ast_signature, expected_labels.join("|"));
150            assert_eq!(python_clone_parse_count_for_test(), 1);
151            if source.contains("compare") {
152                for expected in ["STR", "NUM", "BOOL", "OP:="] {
153                    assert!(
154                        preparation
155                            .normalized_tokens
156                            .iter()
157                            .any(|token| token == expected),
158                        "missing normalized token {expected}"
159                    );
160                }
161                assert!(
162                    preparation
163                        .normalized_tokens
164                        .iter()
165                        .all(|token| token != "T:comment")
166                );
167                assert!(
168                    preparation
169                        .ast_signature
170                        .split('|')
171                        .any(|label| label == "N:comment")
172                );
173            }
174        }
175    }
176
177    #[test]
178    #[hardwon("#3539: rejected clone candidates repeated source parsing")]
179    fn rejected_candidate_still_uses_one_parse() {
180        let token_count = prepare_python_clone("value = 1", 0)
181            .unwrap()
182            .normalized_tokens
183            .len();
184        reset_python_clone_parse_count_for_test();
185
186        assert!(prepare_python_clone("value = 1", token_count).is_some());
187        assert_eq!(python_clone_parse_count_for_test(), 1);
188        reset_python_clone_parse_count_for_test();
189
190        assert!(prepare_python_clone("value = 1", token_count + 1).is_none());
191        assert_eq!(python_clone_parse_count_for_test(), 1);
192    }
193
194    #[test]
195    #[hardwon("#3539: clone preparation recursively overflowed on deeply nested syntax")]
196    fn deeply_nested_source_is_walked_without_recursion() {
197        let depth = 2_000;
198        let source = format!("value = {}0{}\n", "(".repeat(depth), ")".repeat(depth));
199
200        let preparation = prepare_python_clone(&source, 0).unwrap();
201        brokk_bifrost_core::test_support::assert_deep_clone_preparation(
202            &preparation.normalized_tokens,
203            &preparation.ast_signature,
204            depth,
205        );
206    }
207}
brokk_bifrost_python/clones.rs

brokk_bifrost_python/
clones.rs
