use std::time::{Instant, SystemTime, UNIX_EPOCH};
use tree_sitter::{Node as TsNode, Parser, Tree};
use crate::extraction::complexity::{count_complexity, PYTHON_COMPLEXITY};
use crate::types::{
generate_node_id, Edge, EdgeKind, ExtractionResult, Node, NodeKind, UnresolvedRef, Visibility,
};
pub struct PythonExtractor;
struct ExtractionState {
nodes: Vec<Node>,
edges: Vec<Edge>,
unresolved_refs: Vec<UnresolvedRef>,
errors: Vec<String>,
node_stack: Vec<(String, String)>,
file_path: String,
source: Vec<u8>,
timestamp: u64,
class_depth: usize,
}
impl ExtractionState {
fn new(file_path: &str, source: &str) -> Self {
let timestamp = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs();
Self {
nodes: Vec::new(),
edges: Vec::new(),
unresolved_refs: Vec::new(),
errors: Vec::new(),
node_stack: Vec::new(),
file_path: file_path.to_string(),
source: source.as_bytes().to_vec(),
timestamp,
class_depth: 0,
}
}
fn qualified_prefix(&self) -> String {
let mut parts = vec![self.file_path.clone()];
for (name, _) in &self.node_stack {
parts.push(name.clone());
}
parts.join("::")
}
fn parent_node_id(&self) -> Option<&str> {
self.node_stack.last().map(|(_, id)| id.as_str())
}
fn node_text(&self, node: TsNode<'_>) -> String {
node.utf8_text(&self.source)
.unwrap_or("<invalid utf8>")
.to_string()
}
}
impl PythonExtractor {
pub fn extract_python(file_path: &str, source: &str) -> ExtractionResult {
let start = Instant::now();
let mut state = ExtractionState::new(file_path, source);
let tree = match Self::parse_source(source) {
Ok(tree) => tree,
Err(msg) => {
state.errors.push(msg);
return Self::build_result(state, start);
}
};
let file_node = Node {
id: generate_node_id(file_path, &NodeKind::File, file_path, 0),
kind: NodeKind::File,
name: file_path.to_string(),
qualified_name: file_path.to_string(),
file_path: file_path.to_string(),
start_line: 0,
end_line: source.lines().count().saturating_sub(1) as u32,
start_column: 0,
end_column: 0,
signature: None,
docstring: None,
visibility: Visibility::Pub,
is_async: false,
branches: 0,
loops: 0,
returns: 0,
max_nesting: 0,
unsafe_blocks: 0,
unchecked_calls: 0,
assertions: 0,
updated_at: state.timestamp,
};
let file_node_id = file_node.id.clone();
state.nodes.push(file_node);
state.node_stack.push((file_path.to_string(), file_node_id));
let root = tree.root_node();
Self::visit_children(&mut state, root);
state.node_stack.pop();
Self::build_result(state, start)
}
fn parse_source(source: &str) -> Result<Tree, String> {
let mut parser = Parser::new();
let language = crate::extraction::ts_provider::language("python");
parser
.set_language(&language)
.map_err(|e| format!("failed to load Python grammar: {e}"))?;
parser
.parse(source, None)
.ok_or_else(|| "tree-sitter parse returned None".to_string())
}
fn visit_children(state: &mut ExtractionState, node: TsNode<'_>) {
let mut cursor = node.walk();
if cursor.goto_first_child() {
loop {
let child = cursor.node();
Self::visit_node(state, child);
if !cursor.goto_next_sibling() {
break;
}
}
}
}
fn visit_node(state: &mut ExtractionState, node: TsNode<'_>) {
match node.kind() {
"function_definition" => {
let is_async = Self::has_async_keyword(node);
Self::visit_function(state, node, is_async);
}
"class_definition" => Self::visit_class(state, node),
"decorated_definition" => Self::visit_decorated_definition(state, node),
"import_statement" => Self::visit_import(state, node),
"import_from_statement" => Self::visit_import_from(state, node),
"expression_statement" if state.class_depth == 0 => {
let mut cursor = node.walk();
if cursor.goto_first_child() {
loop {
let child = cursor.node();
if child.kind() == "assignment" {
Self::visit_assignment(state, child);
}
if !cursor.goto_next_sibling() {
break;
}
}
}
}
_ => {}
}
}
fn visit_function(state: &mut ExtractionState, node: TsNode<'_>, is_async: bool) {
let name = Self::find_child_by_kind(node, "identifier")
.map_or_else(|| "<anonymous>".to_string(), |n| state.node_text(n));
let in_class = state.class_depth > 0;
let kind = if in_class {
NodeKind::Method
} else {
NodeKind::Function
};
let visibility = Self::python_visibility(&name);
let signature = Some(Self::extract_function_signature(state, node));
let docstring = Self::extract_docstring(state, node);
let start_line = node.start_position().row as u32;
let end_line = node.end_position().row as u32;
let start_column = node.start_position().column as u32;
let end_column = node.end_position().column as u32;
let qualified_name = format!("{}::{}", state.qualified_prefix(), name);
let id = generate_node_id(&state.file_path, &kind, &name, start_line);
let metrics = count_complexity(node, &PYTHON_COMPLEXITY, &state.source);
let graph_node = Node {
id: id.clone(),
kind,
name: name.clone(),
qualified_name,
file_path: state.file_path.clone(),
start_line,
end_line,
start_column,
end_column,
signature,
docstring,
visibility,
is_async,
branches: metrics.branches,
loops: metrics.loops,
returns: metrics.returns,
max_nesting: metrics.max_nesting,
unsafe_blocks: metrics.unsafe_blocks,
unchecked_calls: metrics.unchecked_calls,
assertions: metrics.assertions,
updated_at: state.timestamp,
};
state.nodes.push(graph_node);
if let Some(parent_id) = state.parent_node_id() {
state.edges.push(Edge {
source: parent_id.to_string(),
target: id.clone(),
kind: EdgeKind::Contains,
line: Some(start_line),
});
}
if let Some(body) = Self::find_child_by_kind(node, "block") {
Self::extract_call_sites(state, body, &id);
}
}
fn visit_class(state: &mut ExtractionState, node: TsNode<'_>) {
let name = Self::find_child_by_kind(node, "identifier")
.map_or_else(|| "<anonymous>".to_string(), |n| state.node_text(n));
let visibility = Self::python_visibility(&name);
let docstring = Self::extract_docstring(state, node);
let signature = Some(Self::extract_class_signature(state, node));
let start_line = node.start_position().row as u32;
let end_line = node.end_position().row as u32;
let start_column = node.start_position().column as u32;
let end_column = node.end_position().column as u32;
let qualified_name = format!("{}::{}", state.qualified_prefix(), name);
let id = generate_node_id(&state.file_path, &NodeKind::Class, &name, start_line);
let graph_node = Node {
id: id.clone(),
kind: NodeKind::Class,
name: name.clone(),
qualified_name,
file_path: state.file_path.clone(),
start_line,
end_line,
start_column,
end_column,
signature,
docstring,
visibility,
is_async: false,
branches: 0,
loops: 0,
returns: 0,
max_nesting: 0,
unsafe_blocks: 0,
unchecked_calls: 0,
assertions: 0,
updated_at: state.timestamp,
};
state.nodes.push(graph_node);
if let Some(parent_id) = state.parent_node_id() {
state.edges.push(Edge {
source: parent_id.to_string(),
target: id.clone(),
kind: EdgeKind::Contains,
line: Some(start_line),
});
}
Self::extract_base_classes(state, node, &id);
state.node_stack.push((name.clone(), id));
state.class_depth += 1;
if let Some(body) = Self::find_child_by_kind(node, "block") {
Self::visit_children(state, body);
}
state.class_depth -= 1;
state.node_stack.pop();
}
fn visit_decorated_definition(state: &mut ExtractionState, node: TsNode<'_>) {
let inner_def = Self::find_child_by_kind(node, "function_definition")
.or_else(|| Self::find_child_by_kind(node, "class_definition"));
let is_async = Self::has_async_keyword(node);
let inner_kind_and_name = if let Some(inner) = inner_def {
let name = Self::find_child_by_kind(inner, "identifier")
.map_or_else(|| "<anonymous>".to_string(), |n| state.node_text(n));
let kind = match inner.kind() {
"class_definition" => NodeKind::Class,
_ => {
if state.class_depth > 0 {
NodeKind::Method
} else {
NodeKind::Function
}
}
};
let start_line = inner.start_position().row as u32;
Some((kind, name, start_line))
} else {
None
};
let mut cursor = node.walk();
if cursor.goto_first_child() {
loop {
let child = cursor.node();
if child.kind() == "decorator" {
let text = state.node_text(child);
let raw = text.trim_start_matches('@');
let name = raw.split('(').next().unwrap_or(raw).trim().to_string();
let start_line = child.start_position().row as u32;
let end_line = child.end_position().row as u32;
let start_column = child.start_position().column as u32;
let end_column = child.end_position().column as u32;
let qualified_name = format!("{}::@{}", state.qualified_prefix(), name);
let dec_id =
generate_node_id(&state.file_path, &NodeKind::Decorator, &name, start_line);
let graph_node = Node {
id: dec_id.clone(),
kind: NodeKind::Decorator,
name: name.clone(),
qualified_name,
file_path: state.file_path.clone(),
start_line,
end_line,
start_column,
end_column,
signature: Some(text),
docstring: None,
visibility: Visibility::Private,
is_async: false,
branches: 0,
loops: 0,
returns: 0,
max_nesting: 0,
unsafe_blocks: 0,
unchecked_calls: 0,
assertions: 0,
updated_at: state.timestamp,
};
state.nodes.push(graph_node);
if let Some((ref kind, ref inner_name, inner_line)) = inner_kind_and_name {
let target_id =
generate_node_id(&state.file_path, kind, inner_name, inner_line);
state.edges.push(Edge {
source: dec_id,
target: target_id,
kind: EdgeKind::Annotates,
line: Some(start_line),
});
}
}
if !cursor.goto_next_sibling() {
break;
}
}
}
if let Some(inner) = inner_def {
match inner.kind() {
"function_definition" => Self::visit_function(state, inner, is_async),
"class_definition" => Self::visit_class(state, inner),
_ => {}
}
}
}
fn visit_import(state: &mut ExtractionState, node: TsNode<'_>) {
let mut cursor = node.walk();
if cursor.goto_first_child() {
loop {
let child = cursor.node();
if child.kind() == "dotted_name" || child.kind() == "aliased_import" {
let import_name = if child.kind() == "aliased_import" {
Self::find_child_by_kind(child, "dotted_name")
.map_or_else(|| state.node_text(child), |n| state.node_text(n))
} else {
state.node_text(child)
};
Self::create_use_node(state, &import_name, node);
}
if !cursor.goto_next_sibling() {
break;
}
}
}
}
fn visit_import_from(state: &mut ExtractionState, node: TsNode<'_>) {
let module_name = Self::find_child_by_kind(node, "dotted_name")
.or_else(|| Self::find_child_by_kind(node, "relative_import"))
.map(|n| state.node_text(n))
.unwrap_or_default();
let mut found_names = false;
if Self::find_child_by_kind(node, "wildcard_import").is_some() {
let full_name = format!("{module_name}.*");
Self::create_use_node(state, &full_name, node);
return;
}
let mut cursor = node.walk();
if cursor.goto_first_child() {
loop {
let child = cursor.node();
if child.kind() == "aliased_import" {
let import_name = Self::find_child_by_kind(child, "dotted_name")
.map_or_else(|| state.node_text(child), |n| state.node_text(n));
let full_name = if module_name.is_empty() {
import_name
} else {
format!("{module_name}.{import_name}")
};
Self::create_use_node(state, &full_name, node);
found_names = true;
}
if !cursor.goto_next_sibling() {
break;
}
}
}
if !found_names {
Self::extract_from_import_names(state, node, &module_name);
}
}
fn extract_from_import_names(state: &mut ExtractionState, node: TsNode<'_>, module_name: &str) {
let mut cursor = node.walk();
let mut past_import_keyword = false;
if cursor.goto_first_child() {
loop {
let child = cursor.node();
if child.kind() == "import" {
past_import_keyword = true;
} else if past_import_keyword {
match child.kind() {
"dotted_name" => {
let import_name = state.node_text(child);
let full_name = if module_name.is_empty() {
import_name
} else {
format!("{module_name}.{import_name}")
};
Self::create_use_node(state, &full_name, node);
}
"aliased_import" => {
let import_name = Self::find_child_by_kind(child, "dotted_name")
.map_or_else(|| state.node_text(child), |n| state.node_text(n));
let full_name = if module_name.is_empty() {
import_name
} else {
format!("{module_name}.{import_name}")
};
Self::create_use_node(state, &full_name, node);
}
_ => {}
}
}
if !cursor.goto_next_sibling() {
break;
}
}
}
}
fn create_use_node(state: &mut ExtractionState, name: &str, node: TsNode<'_>) {
let start_line = node.start_position().row as u32;
let end_line = node.end_position().row as u32;
let start_column = node.start_position().column as u32;
let end_column = node.end_position().column as u32;
let qualified_name = format!("{}::{}", state.qualified_prefix(), name);
let id = generate_node_id(&state.file_path, &NodeKind::Use, name, start_line);
let graph_node = Node {
id: id.clone(),
kind: NodeKind::Use,
name: name.to_string(),
qualified_name,
file_path: state.file_path.clone(),
start_line,
end_line,
start_column,
end_column,
signature: Some(state.node_text(node).trim().to_string()),
docstring: None,
visibility: Visibility::Private,
is_async: false,
branches: 0,
loops: 0,
returns: 0,
max_nesting: 0,
unsafe_blocks: 0,
unchecked_calls: 0,
assertions: 0,
updated_at: state.timestamp,
};
state.nodes.push(graph_node);
if let Some(parent_id) = state.parent_node_id() {
state.edges.push(Edge {
source: parent_id.to_string(),
target: id.clone(),
kind: EdgeKind::Contains,
line: Some(start_line),
});
}
state.unresolved_refs.push(UnresolvedRef {
from_node_id: id,
reference_name: name.to_string(),
reference_kind: EdgeKind::Uses,
line: start_line,
column: start_column,
file_path: state.file_path.clone(),
});
}
fn visit_assignment(state: &mut ExtractionState, node: TsNode<'_>) {
let left = node.child_by_field_name("left");
if let Some(left_node) = left {
let name = state.node_text(left_node);
if Self::is_upper_snake_case(&name) {
let start_line = node.start_position().row as u32;
let end_line = node.end_position().row as u32;
let start_column = node.start_position().column as u32;
let end_column = node.end_position().column as u32;
let text = state.node_text(node);
let qualified_name = format!("{}::{}", state.qualified_prefix(), name);
let id = generate_node_id(&state.file_path, &NodeKind::Const, &name, start_line);
let graph_node = Node {
id: id.clone(),
kind: NodeKind::Const,
name,
qualified_name,
file_path: state.file_path.clone(),
start_line,
end_line,
start_column,
end_column,
signature: Some(text.trim().to_string()),
docstring: None,
visibility: Visibility::Pub,
is_async: false,
branches: 0,
loops: 0,
returns: 0,
max_nesting: 0,
unsafe_blocks: 0,
unchecked_calls: 0,
assertions: 0,
updated_at: state.timestamp,
};
state.nodes.push(graph_node);
if let Some(parent_id) = state.parent_node_id() {
state.edges.push(Edge {
source: parent_id.to_string(),
target: id,
kind: EdgeKind::Contains,
line: Some(start_line),
});
}
}
}
}
fn extract_base_classes(state: &mut ExtractionState, node: TsNode<'_>, class_id: &str) {
if let Some(arg_list) = Self::find_child_by_kind(node, "argument_list") {
let mut cursor = arg_list.walk();
if cursor.goto_first_child() {
loop {
let child = cursor.node();
match child.kind() {
"identifier" => {
let base_name = state.node_text(child);
let line = child.start_position().row as u32;
let column = child.start_position().column as u32;
state.unresolved_refs.push(UnresolvedRef {
from_node_id: class_id.to_string(),
reference_name: base_name,
reference_kind: EdgeKind::Extends,
line,
column,
file_path: state.file_path.clone(),
});
}
"attribute" => {
let base_name = state.node_text(child);
let line = child.start_position().row as u32;
let column = child.start_position().column as u32;
state.unresolved_refs.push(UnresolvedRef {
from_node_id: class_id.to_string(),
reference_name: base_name,
reference_kind: EdgeKind::Extends,
line,
column,
file_path: state.file_path.clone(),
});
}
_ => {}
}
if !cursor.goto_next_sibling() {
break;
}
}
}
}
}
fn extract_function_signature(state: &ExtractionState, node: TsNode<'_>) -> String {
if let Some(block) = Self::find_child_by_kind(node, "block") {
let text = state.node_text(node);
let block_offset = block.start_byte() - node.start_byte();
let before_block = &text[..block_offset];
before_block.trim().trim_end_matches(':').trim().to_string()
} else {
state.node_text(node).trim().to_string()
}
}
fn extract_class_signature(state: &ExtractionState, node: TsNode<'_>) -> String {
if let Some(block) = Self::find_child_by_kind(node, "block") {
let text = state.node_text(node);
let block_offset = block.start_byte() - node.start_byte();
let before_block = &text[..block_offset];
before_block.trim().trim_end_matches(':').trim().to_string()
} else {
state.node_text(node).trim().to_string()
}
}
fn extract_docstring(state: &ExtractionState, node: TsNode<'_>) -> Option<String> {
let body = Self::find_child_by_kind(node, "block")?;
let mut cursor = body.walk();
if cursor.goto_first_child() {
loop {
let child = cursor.node();
if child.kind() == "expression_statement" {
if let Some(string_node) = Self::find_child_by_kind(child, "string") {
let text = state.node_text(string_node);
return Some(Self::strip_docstring_quotes(&text));
}
return None;
}
if child.kind() != "comment" {
return None;
}
if !cursor.goto_next_sibling() {
break;
}
}
}
None
}
fn strip_docstring_quotes(text: &str) -> String {
let trimmed = text.trim();
if trimmed.starts_with("\"\"\"") && trimmed.ends_with("\"\"\"") && trimmed.len() >= 6 {
return trimmed[3..trimmed.len() - 3].trim().to_string();
}
if trimmed.starts_with("'''") && trimmed.ends_with("'''") && trimmed.len() >= 6 {
return trimmed[3..trimmed.len() - 3].trim().to_string();
}
if (trimmed.starts_with('"') && trimmed.ends_with('"'))
|| (trimmed.starts_with('\'') && trimmed.ends_with('\''))
{
return trimmed[1..trimmed.len() - 1].trim().to_string();
}
trimmed.to_string()
}
fn has_async_keyword(node: TsNode<'_>) -> bool {
let mut cursor = node.walk();
if cursor.goto_first_child() {
loop {
let child = cursor.node();
if child.kind() == "async" {
return true;
}
if child.kind() == "function_definition" {
return Self::has_async_keyword(child);
}
if !cursor.goto_next_sibling() {
break;
}
}
}
false
}
fn extract_call_sites(state: &mut ExtractionState, node: TsNode<'_>, fn_node_id: &str) {
let mut cursor = node.walk();
if cursor.goto_first_child() {
loop {
let child = cursor.node();
match child.kind() {
"call" => {
let callee = child.named_child(0);
if let Some(callee) = callee {
let callee_name = state.node_text(callee);
state.unresolved_refs.push(UnresolvedRef {
from_node_id: fn_node_id.to_string(),
reference_name: callee_name,
reference_kind: EdgeKind::Calls,
line: child.start_position().row as u32,
column: child.start_position().column as u32,
file_path: state.file_path.clone(),
});
}
Self::extract_call_sites(state, child, fn_node_id);
}
"function_definition" | "class_definition" => {}
_ => {
Self::extract_call_sites(state, child, fn_node_id);
}
}
if !cursor.goto_next_sibling() {
break;
}
}
}
}
fn python_visibility(name: &str) -> Visibility {
if name.starts_with("__") && name.ends_with("__") && name.len() > 4 {
Visibility::Pub } else if name.starts_with('_') {
Visibility::Private } else {
Visibility::Pub
}
}
fn is_upper_snake_case(name: &str) -> bool {
if name.is_empty() {
return false;
}
let has_upper = name.chars().any(|c| c.is_ascii_uppercase());
let all_valid = name
.chars()
.all(|c| c.is_ascii_uppercase() || c.is_ascii_digit() || c == '_');
let starts_ok = !name.starts_with(|c: char| c.is_ascii_digit());
has_upper && all_valid && starts_ok
}
fn find_child_by_kind<'a>(node: TsNode<'a>, kind: &str) -> Option<TsNode<'a>> {
let mut cursor = node.walk();
if cursor.goto_first_child() {
loop {
let child = cursor.node();
if child.kind() == kind {
return Some(child);
}
if !cursor.goto_next_sibling() {
break;
}
}
}
None
}
fn build_result(state: ExtractionState, start: Instant) -> ExtractionResult {
ExtractionResult {
nodes: state.nodes,
edges: state.edges,
unresolved_refs: state.unresolved_refs,
errors: state.errors,
duration_ms: start.elapsed().as_millis() as u64,
}
}
}
impl crate::extraction::LanguageExtractor for PythonExtractor {
fn extensions(&self) -> &[&str] {
&["py"]
}
fn language_name(&self) -> &'static str {
"Python"
}
fn extract(&self, file_path: &str, source: &str) -> ExtractionResult {
PythonExtractor::extract_python(file_path, source)
}
}