brokk_bifrost_python/
clones.rs

1//! Python's clone-detection token and AST-signature normalization.
2//!
3//! `analyzer/python/clones.rs` in `brokk-bifrost-analysis` keeps the entry
4//! point: it reads the declaration's source through the analyzer and assembles
5//! the analysis-owned `CloneCandidateData`. Everything that knows what a Python
6//! token *is* lives here.
7
8use crate::declarations::parse_python_tree;
9use tree_sitter::Node;
10
11const PYTHON_CLONE_AST_IDENTIFIER_TYPES: &[&str] = &["identifier", "keyword_identifier"];
12const PYTHON_CLONE_AST_STRING_TYPES: &[&str] = &["string", "string_content", "interpolation"];
13const PYTHON_CLONE_AST_NUMBER_TYPES: &[&str] = &["integer", "float"];
14
15pub fn normalized_clone_tokens_python(source: &str) -> Vec<String> {
16    let Some(tree) = parse_python_tree(source) else {
17        return Vec::new();
18    };
19    let mut out = Vec::new();
20    collect_normalized_leaf_tokens_python(tree.root_node(), source, &mut out);
21    out
22}
23
24fn collect_normalized_leaf_tokens_python(node: Node<'_>, source: &str, out: &mut Vec<String>) {
25    if node.named_child_count() == 0 {
26        let token = normalize_python_clone_leaf_token(node, source);
27        if !token.is_empty() {
28            out.push(token);
29        }
30    }
31    let child_count = node.child_count();
32    for index in 0..child_count {
33        if let Some(child) = node.child(index) {
34            collect_normalized_leaf_tokens_python(child, source, out);
35        }
36    }
37}
38
39fn normalize_python_clone_leaf_token(node: Node<'_>, source: &str) -> String {
40    let kind = node.kind();
41    let token = source
42        .get(node.start_byte()..node.end_byte())
43        .unwrap_or("")
44        .trim();
45    if token.is_empty() || kind == "comment" {
46        return String::new();
47    }
48    if PYTHON_CLONE_AST_IDENTIFIER_TYPES.contains(&kind) {
49        return "ID".to_string();
50    }
51    if PYTHON_CLONE_AST_STRING_TYPES.contains(&kind) {
52        return "STR".to_string();
53    }
54    if PYTHON_CLONE_AST_NUMBER_TYPES.contains(&kind) {
55        return "NUM".to_string();
56    }
57    if kind == "true" || kind == "false" || token == "True" || token == "False" {
58        return "BOOL".to_string();
59    }
60    if token.chars().count() == 1 && token.chars().all(|ch| !ch.is_alphanumeric()) {
61        return format!("OP:{token}");
62    }
63    format!("T:{kind}")
64}
65
66pub fn build_python_clone_ast_signature(source: &str) -> String {
67    let Some(tree) = parse_python_tree(source) else {
68        return String::new();
69    };
70    let mut labels = Vec::new();
71    collect_python_clone_ast_labels(tree.root_node(), source, &mut labels);
72    labels.join("|")
73}
74
75fn collect_python_clone_ast_labels(node: Node<'_>, source: &str, out: &mut Vec<String>) {
76    out.push(normalize_python_clone_ast_label(node, source));
77    let child_count = node.child_count();
78    for index in 0..child_count {
79        if let Some(child) = node.child(index) {
80            collect_python_clone_ast_labels(child, source, out);
81        }
82    }
83}
84
85fn normalize_python_clone_ast_label(node: Node<'_>, source: &str) -> String {
86    let kind = node.kind();
87    let text = source
88        .get(node.start_byte()..node.end_byte())
89        .unwrap_or("")
90        .trim();
91    if PYTHON_CLONE_AST_IDENTIFIER_TYPES.contains(&kind) {
92        return "ID".to_string();
93    }
94    if PYTHON_CLONE_AST_STRING_TYPES.contains(&kind) {
95        return "STR".to_string();
96    }
97    if PYTHON_CLONE_AST_NUMBER_TYPES.contains(&kind) {
98        return "NUM".to_string();
99    }
100    if kind == "true" || kind == "false" || text == "True" || text == "False" {
101        return "BOOL".to_string();
102    }
103    format!("N:{kind}")
104}
brokk_bifrost_python/clones.rs

brokk_bifrost_python/
clones.rs
