1use crate::bindings::python_direct_scope_bindings_bounded;
2use crate::imports::{python_import_infos_from_node, python_import_syntaxes_from_node};
3use crate::syntax::{
4 PythonOverloadDecoratorBindings, PythonOverloadDecoratorName, expression_name_node,
5 python_parameter_name, python_plain_string_literal,
6};
7use brokk_bifrost_core::analyzer::fq_name::{FqName, SegmentKind, segment_interner};
8use brokk_bifrost_core::analyzer::model::{
9 CallableArity, CodeUnitType, DispatchExtensibility, ParameterMetadata, SignatureMetadata,
10 StructuredImportPathKind, StructuredTypeIdentity,
11};
12use brokk_bifrost_core::analyzer::parsed_file::{
13 ParsedFile, ParsedSourceFacts, SourceDeclarationMetadataLink, SourceImportFact,
14};
15use brokk_bifrost_core::analyzer::project::Project;
16use brokk_bifrost_core::analyzer::python_facts::{PythonCallableReturnFact, PythonSourceFacts};
17use brokk_bifrost_core::analyzer::resolution_facts::ResolutionSiteId;
18use brokk_bifrost_core::analyzer::rust_facts::RustItemSourceFacts;
19use brokk_bifrost_core::analyzer::source_facts::{
20 PrimarySourceFactCollector, SourceDeclarationId, SourceImportId, SourceOccurrenceId,
21};
22use brokk_bifrost_core::analyzer::structural::callable::CallSiteContext;
23use brokk_bifrost_core::analyzer::structural::collector::StructuralFactCollector;
24use brokk_bifrost_core::analyzer::structural::resolution::DeclaredVisibility;
25use brokk_bifrost_core::analyzer::structural::spec::{CompiledKinds, StructuralSpec};
26use brokk_bifrost_core::analyzer::tree_walk::{ParentIndex, WalkControl, walk_named_tree_preorder};
27use brokk_bifrost_core::analyzer::{CodeUnit, ProjectFile};
28use brokk_bifrost_core::hash::{HashMap, HashSet};
29use brokk_bifrost_core::path_normalization::NormalizePath;
30use brokk_bifrost_core::text_utils::{compute_line_starts, find_line_index_for_offset};
31use std::borrow::Cow;
32use std::path::{Path, PathBuf};
33use tree_sitter::{Node, Parser, Tree};
34
35use crate::source_properties::{python_annotation_references, python_return_type_identity};
36use crate::structural::{PYTHON_STRUCTURAL_SPEC, python_parenthesized_expression_child};
37
38pub fn python_module_fq(project: &dyn Project, file: &ProjectFile) -> FqName {
51 FqName::from_package_components(&python_module_components(project, file))
52}
53
54fn python_module_components(project: &dyn Project, file: &ProjectFile) -> Vec<String> {
55 python_module_components_from_root(file, &python_import_root(project, file))
56}
57
58pub fn python_module_components_from_name(file: &ProjectFile, name: &str) -> Option<Vec<String>> {
62 let components = python_project_module_components(file);
63 (0..components.len()).find_map(|start| {
64 (components[start..].join(".") == name).then(|| components[start..].to_vec())
65 })
66}
67
68pub fn python_project_module_components(file: &ProjectFile) -> Vec<String> {
80 python_module_components_from_root(file, Path::new(""))
81}
82
83fn python_module_components_from_root(file: &ProjectFile, import_root_rel: &Path) -> Vec<String> {
86 let mut components = python_package_components_for_file(file, import_root_rel);
87 let module_name = file
88 .rel_path()
89 .file_stem()
90 .and_then(|stem| stem.to_str())
91 .unwrap_or_default();
92 if module_name != "__init__" || components.is_empty() {
93 components.push(module_name.to_string());
94 }
95 components
96}
97
98pub trait PythonImportRootInputs {
100 fn contains(&self, relative_path: &Path) -> bool;
101 fn configuration_source(&self, relative_path: &Path) -> Option<Cow<'_, str>>;
102}
103
104struct ProjectImportRootInputs<'a>(&'a dyn Project);
105impl PythonImportRootInputs for ProjectImportRootInputs<'_> {
106 fn contains(&self, relative_path: &Path) -> bool {
107 self.0.file_by_rel_path(relative_path).is_some()
108 }
109 fn configuration_source(&self, relative_path: &Path) -> Option<Cow<'_, str>> {
110 let file = self.0.file_by_rel_path(relative_path)?;
111 self.0.read_source(&file).ok().map(Cow::Owned)
112 }
113}
114
115pub fn python_import_root(project: &dyn Project, file: &ProjectFile) -> PathBuf {
127 python_import_root_with_inputs(
128 file.root(),
129 file.rel_path(),
130 &ProjectImportRootInputs(project),
131 )
132}
133
134pub fn python_import_root_with_inputs(
136 root: &Path,
137 relative_path: &Path,
138 inputs: &impl PythonImportRootInputs,
139) -> PathBuf {
140 let Some(parent_rel) = relative_path.parent() else {
141 return PathBuf::new();
142 };
143 if parent_rel.as_os_str().is_empty() {
144 return PathBuf::new();
145 }
146 if let Some(import_root_rel) = python_configured_import_root(root, parent_rel, inputs) {
147 return import_root_rel;
148 }
149 let mut effective_package_root_rel: Option<&Path> = None;
150 for path in parent_rel.ancestors() {
151 if inputs.contains(&path.join("__init__.py")) {
152 effective_package_root_rel = Some(path);
153 }
154 }
155 effective_package_root_rel
156 .and_then(Path::parent)
157 .unwrap_or_else(|| Path::new(""))
158 .to_path_buf()
159}
160
161fn python_package_components_for_file(file: &ProjectFile, import_root_rel: &Path) -> Vec<String> {
162 let Some(parent_rel) = file.rel_path().parent() else {
163 return Vec::new();
164 };
165 let relative_package = parent_rel
166 .strip_prefix(import_root_rel)
167 .expect("a Python import root is a prefix of the paths it names modules for");
168 path_components(relative_package)
169}
170
171pub const PYTHON_CONFIGURATION_INPUT_NAMES: [&str; 3] =
173 ["pyproject.toml", "setup.py", "__init__.py"];
174
175fn python_configured_import_root(
181 workspace_root: &Path,
182 parent_rel: &Path,
183 inputs: &impl PythonImportRootInputs,
184) -> Option<PathBuf> {
185 let mut manifest_dir_rel = Some(parent_rel);
186 while let Some(directory) = manifest_dir_rel {
187 let manifest_dir = workspace_root.join(directory);
188 let mut roots = inputs
189 .configuration_source(&directory.join("pyproject.toml"))
190 .map(|source| parse_setuptools_where_entries(&source))
191 .unwrap_or_default()
192 .iter()
193 .map(|entry| manifest_dir.join(entry).normalize())
194 .filter_map(|root| {
195 root.strip_prefix(workspace_root)
196 .ok()
197 .map(Path::to_path_buf)
198 })
199 .filter(|root| parent_rel.starts_with(root))
200 .collect::<Vec<_>>();
201 roots.sort_by_key(|root| root.components().count());
202 if let Some(root) = roots.pop() {
203 return Some(root);
204 }
205 if let Some(package_dir) = inputs
206 .configuration_source(&directory.join("setup.py"))
207 .and_then(|source| parse_setuptools_setup_py_import_root(&source))
208 {
209 let root = manifest_dir.join(package_dir).normalize();
210 if let Ok(root) = root.strip_prefix(workspace_root)
211 && parent_rel.starts_with(root)
212 {
213 return Some(root.to_path_buf());
214 }
215 }
216 manifest_dir_rel = directory.parent();
217 }
218 None
219}
220
221pub fn python_declared_import_root(project: &dyn Project, file: &ProjectFile) -> Option<PathBuf> {
233 let parent_rel = file.rel_path().parent()?;
234 if parent_rel.as_os_str().is_empty() {
235 return None;
236 }
237 python_configured_import_root(file.root(), parent_rel, &ProjectImportRootInputs(project))
238}
239
240pub(crate) fn python_repeated_root_extension_dir(
265 file: &ProjectFile,
266 source_root_rel: &Path,
267 declared_root_rel: &Path,
268) -> Option<PathBuf> {
269 let implied_rel = source_root_rel.strip_prefix(declared_root_rel).ok()?;
270 let implied = path_components(implied_rel);
271 if implied.is_empty()
272 || !python_module_components_from_root(file, source_root_rel).starts_with(&implied)
273 {
274 return None;
275 }
276 Some(source_root_rel.join(implied_rel))
277}
278
279fn parse_setuptools_setup_py_import_root(source: &str) -> Option<PathBuf> {
285 let tree = parse_python_tree(source)?;
286 let root = tree.root_node();
287 if root.has_error() {
288 return None;
289 }
290 let mut import_root = None;
291 for (call, setup_bindings) in setup_py_setup_calls(source, root) {
292 let candidate = setup_py_import_root_from_call(call, source, &setup_bindings)?;
293 if import_root
294 .replace(candidate.clone())
295 .is_some_and(|root| root != candidate)
296 {
297 return None;
298 }
299 }
300 import_root
301}
302
303pub fn setuptools_setup_py_python_requires(setup_py: &Path) -> Option<String> {
311 let source = std::fs::read_to_string(setup_py).ok()?;
312 setuptools_setup_py_python_requires_from_source(&source)
313}
314
315pub fn setuptools_setup_py_python_requires_from_source(source: &str) -> Option<String> {
317 let tree = parse_python_tree(source)?;
318 let root = tree.root_node();
319 if root.has_error() {
320 return None;
321 }
322 let mut requirement: Option<String> = None;
323 for (call, _) in setup_py_setup_calls(source, root) {
324 let Some(arguments) = call.child_by_field_name("arguments") else {
325 continue;
326 };
327 if arguments.kind() != "argument_list" {
328 continue;
329 }
330 let mut cursor = arguments.walk();
331 for argument in arguments.named_children(&mut cursor) {
332 if argument.kind() != "keyword_argument" {
333 continue;
334 }
335 let Some(name) = argument.child_by_field_name("name") else {
336 continue;
337 };
338 if py_node_text(name, source).trim() != "python_requires" {
339 continue;
340 }
341 let value = argument.child_by_field_name("value")?;
342 let declared = python_plain_string_literal(value, source)?.to_owned();
343 if requirement
344 .replace(declared.clone())
345 .is_some_and(|previous| previous != declared)
346 {
347 return None;
348 }
349 }
350 }
351 requirement
352}
353
354#[derive(Debug, Clone)]
357pub struct SetuptoolsLiteralArgument {
358 pub name: String,
359 pub value: Option<SetuptoolsLiteralValue>,
360 pub byte_range: std::ops::Range<usize>,
361 pub call_index: usize,
362}
363
364#[derive(Debug, Clone)]
365pub enum SetuptoolsLiteralValue {
366 String(String),
367 Strings(Vec<String>),
368 Extras(Vec<(String, Vec<String>)>),
369}
370
371pub fn setuptools_setup_py_literal_arguments(
372 source: &str,
373) -> Option<Vec<SetuptoolsLiteralArgument>> {
374 let tree = parse_python_tree(source)?;
375 let root = tree.root_node();
376 if root.has_error() {
377 return None;
378 }
379 let mut output = Vec::new();
380 for (call_index, (call, _)) in setup_py_setup_calls(source, root).into_iter().enumerate() {
381 let arguments = call.child_by_field_name("arguments")?;
382 let mut cursor = arguments.walk();
383 for argument in arguments.named_children(&mut cursor) {
384 if argument.kind() == "dictionary_splat" {
385 output.push(SetuptoolsLiteralArgument {
386 name: "**kwargs".to_owned(),
387 value: None,
388 byte_range: argument.byte_range(),
389 call_index,
390 });
391 continue;
392 }
393 if argument.kind() != "keyword_argument" {
394 continue;
395 }
396 let name = py_node_text(argument.child_by_field_name("name")?, source).to_owned();
397 let node = argument.child_by_field_name("value")?;
398 let value = match name.as_str() {
399 "name" | "version" | "python_requires" => python_plain_string_literal(node, source)
400 .map(|value| SetuptoolsLiteralValue::String(value.to_owned())),
401 "install_requires" | "setup_requires" | "tests_require" => {
402 setuptools_literal_strings(node, source).map(SetuptoolsLiteralValue::Strings)
403 }
404 "extras_require" => {
405 let mut entries = Vec::new();
406 let mut cursor = node.walk();
407 let mut complete = node.kind() == "dictionary";
408 for pair in node.named_children(&mut cursor) {
409 let entry = pair
410 .child_by_field_name("key")
411 .and_then(|key| python_plain_string_literal(key, source))
412 .zip(
413 pair.child_by_field_name("value")
414 .and_then(|value| setuptools_literal_strings(value, source)),
415 );
416 if let Some((key, values)) = entry {
417 entries.push((key.to_owned(), values));
418 } else {
419 complete = false;
420 }
421 }
422 complete.then_some(SetuptoolsLiteralValue::Extras(entries))
423 }
424 _ => continue,
425 };
426 output.push(SetuptoolsLiteralArgument {
427 name,
428 value,
429 byte_range: node.byte_range(),
430 call_index,
431 });
432 }
433 }
434 Some(output)
435}
436
437fn setuptools_literal_strings(node: Node<'_>, source: &str) -> Option<Vec<String>> {
438 if !matches!(node.kind(), "list" | "tuple") {
439 return None;
440 }
441 let mut cursor = node.walk();
442 node.named_children(&mut cursor)
443 .filter(|child| child.kind() != "comment")
444 .map(|child| python_plain_string_literal(child, source).map(str::to_owned))
445 .collect()
446}
447
448fn setup_py_setup_calls<'tree>(
456 source: &str,
457 root: Node<'tree>,
458) -> Vec<(Node<'tree>, HashMap<Vec<String>, String>)> {
459 let mut setup_bindings: HashMap<Vec<String>, String> = HashMap::default();
460 let mut calls = Vec::new();
461 let mut cursor = root.walk();
462
463 for statement in root.named_children(&mut cursor) {
464 if matches!(
465 statement.kind(),
466 "import_statement" | "import_from_statement"
467 ) {
468 for binding in setup_py_bound_names(statement, source) {
469 setup_bindings.retain(|path, _| path.first() != Some(&binding));
470 }
471 for import in python_import_infos_from_node(statement, source) {
472 if import.is_wildcard {
473 setup_bindings.clear();
474 continue;
475 }
476 let Some(path) = import.path else { continue };
477 let segments = path.segments.iter().map(String::as_str).collect::<Vec<_>>();
478 match path.kind {
479 Some(StructuredImportPathKind::ImportFrom) => {
480 let function_name = match segments.as_slice() {
481 ["setuptools", "setup"] | ["distutils", "core", "setup"] => "setup",
482 ["setuptools", "find_packages"] => "find_packages",
483 ["setuptools", "find_namespace_packages"] => "find_namespace_packages",
484 _ => continue,
485 };
486 setup_bindings.insert(
487 vec![import.identifier.expect("imported function binds a name")],
488 function_name.to_string(),
489 );
490 }
491 Some(StructuredImportPathKind::Namespace) => {
492 let function_names = match segments.as_slice() {
493 ["setuptools"] => {
494 ["setup", "find_packages", "find_namespace_packages"].as_slice()
495 }
496 ["distutils", "core"] => ["setup"].as_slice(),
497 _ => continue,
498 };
499 let binding_prefix = import
500 .alias
501 .map(|alias| vec![alias])
502 .unwrap_or_else(|| path.segments.clone());
503 for function_name in function_names {
504 let mut callable = binding_prefix.clone();
505 callable.push((*function_name).to_string());
506 setup_bindings.insert(callable, (*function_name).to_string());
507 }
508 }
509 _ => continue,
510 }
511 }
512 continue;
513 }
514
515 if statement.kind() == "expression_statement"
516 && statement.named_child_count() == 1
517 && let Some(call) = statement.named_child(0)
518 && call.kind() == "call"
519 && setup_py_call_imported_function(call, source, &setup_bindings)
520 .is_some_and(|function| function == "setup")
521 {
522 calls.push((call, setup_bindings.clone()));
523 }
524
525 for binding in setup_py_bound_names(statement, source) {
526 setup_bindings.retain(|path, _| path.first() != Some(&binding));
527 }
528 }
529 calls
530}
531
532fn setup_py_bound_names(statement: Node<'_>, source: &str) -> Vec<String> {
539 let mut names = Vec::new();
540 let mut pending = vec![statement];
541 while let Some(node) = pending.pop() {
542 for binding in python_direct_scope_bindings_bounded(node, source, || true)
543 .expect("unbounded setup.py binding walk")
544 {
545 let name = py_node_text(binding.declaration, source).trim();
546 if !name.is_empty() {
547 names.push(name.to_string());
548 }
549 }
550 let excluded_body = matches!(
551 node.kind(),
552 "function_definition" | "class_definition" | "lambda"
553 )
554 .then(|| node.child_by_field_name("body").map(|body| body.id()))
555 .flatten();
556 let mut cursor = node.walk();
557 pending.extend(
558 node.named_children(&mut cursor)
559 .filter(|child| Some(child.id()) != excluded_body),
560 );
561 }
562 names
563}
564
565fn setup_py_call_imported_function<'a>(
566 call: Node<'_>,
567 source: &str,
568 setup_bindings: &'a HashMap<Vec<String>, String>,
569) -> Option<&'a str> {
570 let mut function = call.child_by_field_name("function")?;
571 let mut path = Vec::new();
572 while function.kind() == "attribute" {
573 let attribute = function.child_by_field_name("attribute")?;
574 path.push(py_node_text(attribute, source).to_string());
575 let object = function.child_by_field_name("object")?;
576 function = object;
577 }
578 if function.kind() != "identifier" {
579 return None;
580 }
581 path.push(py_node_text(function, source).to_string());
582 path.reverse();
583 setup_bindings.get(&path).map(String::as_str)
584}
585
586fn setup_py_import_root_from_call(
587 call: Node<'_>,
588 source: &str,
589 setup_bindings: &HashMap<Vec<String>, String>,
590) -> Option<PathBuf> {
591 let arguments = call.child_by_field_name("arguments")?;
592 if arguments.kind() != "argument_list" {
593 return None;
594 }
595
596 let mut packages = None;
597 let mut package_dir = None;
598 let mut cursor = arguments.walk();
599 for argument in arguments.named_children(&mut cursor) {
600 if argument.kind() == "comment" {
601 continue;
602 }
603 if argument.kind() != "keyword_argument" {
605 return None;
606 }
607 let name = argument.child_by_field_name("name")?;
608 let value = argument.child_by_field_name("value")?;
609 match py_node_text(name, source).trim() {
610 "packages" if packages.is_none() => packages = Some(value),
611 "packages" => return None,
612 "package_dir" if package_dir.is_none() => package_dir = Some(value),
613 "package_dir" => return None,
614 _ => {}
615 }
616 }
617
618 let packages = packages?;
619 if let Some(package_dir) = package_dir {
620 return setup_py_static_package_dir(package_dir, source);
621 }
622 if setup_py_nonempty_literal_packages(packages, source) {
623 return Some(PathBuf::new());
624 }
625 setup_py_discovery_root_from_call(packages, source, setup_bindings)
626}
627
628fn setup_py_discovery_root_from_call(
629 call: Node<'_>,
630 source: &str,
631 setup_bindings: &HashMap<Vec<String>, String>,
632) -> Option<PathBuf> {
633 let function = setup_py_call_imported_function(call, source, setup_bindings)?;
634 if !matches!(function, "find_packages" | "find_namespace_packages") {
635 return None;
636 }
637 let arguments = call.child_by_field_name("arguments")?;
638 if arguments.kind() != "argument_list" {
639 return None;
640 }
641
642 let mut where_value = None;
643 let mut seen_exclude = false;
644 let mut seen_include = false;
645 let mut positional_index = 0;
646 let mut cursor = arguments.walk();
647 for argument in arguments.named_children(&mut cursor) {
648 if argument.kind() == "comment" {
649 continue;
650 }
651 if matches!(argument.kind(), "list_splat" | "dictionary_splat") {
652 return None;
653 }
654 if argument.kind() == "keyword_argument" {
655 let name = py_node_text(argument.child_by_field_name("name")?, source).trim();
656 let value = argument.child_by_field_name("value")?;
657 match name {
658 "where" if where_value.is_none() => where_value = Some(value),
659 "where" => return None,
660 "exclude" if !seen_exclude => seen_exclude = true,
661 "exclude" => return None,
662 "include" if !seen_include => seen_include = true,
663 "include" => return None,
664 _ => return None,
665 }
666 continue;
667 }
668
669 let slot = positional_index;
670 positional_index += 1;
671 match slot {
672 0 if where_value.is_none() => where_value = Some(argument),
673 0 => return None,
674 1 if !seen_exclude => seen_exclude = true,
675 1 => return None,
676 2 if !seen_include => seen_include = true,
677 2 => return None,
678 _ => return None,
679 }
680 }
681
682 where_value
683 .map(|value| python_plain_string_literal(value, source))
684 .unwrap_or(Some(""))
685 .map(PathBuf::from)
686}
687
688fn setup_py_nonempty_literal_packages(value: Node<'_>, source: &str) -> bool {
689 let value = setup_py_unwrap_parenthesized(value);
690 if !matches!(value.kind(), "list" | "set" | "tuple") {
691 return false;
692 }
693 let mut cursor = value.walk();
694 let mut nonempty = false;
695 for element in value.named_children(&mut cursor) {
696 if element.kind() == "comment" {
697 continue;
698 }
699 let Some(package) = python_plain_string_literal(element, source) else {
700 return false;
701 };
702 if package.is_empty() {
703 return false;
704 }
705 nonempty = true;
706 }
707 nonempty
708}
709
710fn setup_py_static_package_dir(value: Node<'_>, source: &str) -> Option<PathBuf> {
711 let value = setup_py_unwrap_parenthesized(value);
712 if value.kind() != "dictionary" {
713 return None;
714 }
715 let mut cursor = value.walk();
716 let mut pairs = value
717 .named_children(&mut cursor)
718 .filter(|node| node.kind() != "comment");
719 let Some(pair) = pairs.next() else {
720 return Some(PathBuf::new());
721 };
722 if pairs.next().is_some() || pair.kind() != "pair" {
723 return None;
724 }
725 let key = python_plain_string_literal(pair.child_by_field_name("key")?, source)?;
726 if !key.is_empty() {
727 return None;
728 }
729 let root = python_plain_string_literal(pair.child_by_field_name("value")?, source)?;
730 let root = PathBuf::from(root);
731 (!root.is_absolute()).then_some(root)
732}
733
734fn setup_py_unwrap_parenthesized(node: Node<'_>) -> Node<'_> {
735 brokk_bifrost_core::analyzer::tree_walk::unwrap_parenthesized_expression_with(
736 node,
737 python_parenthesized_expression_child,
738 )
739}
740
741fn parse_setuptools_where_entries(source: &str) -> Vec<String> {
742 let Ok(document) = source.parse::<toml::Value>() else {
743 return Vec::new();
744 };
745 document
746 .get("tool")
747 .and_then(|tool| tool.get("setuptools"))
748 .and_then(|setuptools| setuptools.get("packages"))
749 .and_then(|packages| packages.get("find"))
750 .and_then(|find| find.get("where"))
751 .and_then(toml::Value::as_array)
752 .map(|entries| {
753 entries
754 .iter()
755 .filter_map(toml::Value::as_str)
756 .map(str::to_string)
757 .collect()
758 })
759 .unwrap_or_default()
760}
761
762fn path_components(path: &Path) -> Vec<String> {
763 path.components()
764 .map(|component| component.as_os_str().to_string_lossy().to_string())
765 .filter(|component| !component.is_empty())
766 .collect()
767}
768
769pub fn python_is_decorated_function_boundary(node: Node<'_>) -> bool {
770 if node.kind() != "decorated_definition" {
771 return false;
772 }
773 let mut cursor = node.walk();
774 node.named_children(&mut cursor)
775 .any(|child| child.kind() == "function_definition")
776}
777
778#[derive(Clone)]
779pub struct Scope {
780 kind: ScopeKind,
781 path: String,
782 fq: FqName,
788 code_unit: Option<CodeUnit>,
789 method_receiver: Option<String>,
790}
791
792#[derive(Clone, Copy, PartialEq, Eq)]
793enum ScopeKind {
794 Class,
795 Function,
796}
797
798pub struct PythonVisitor<'source, 'parsed, 'structural>
799where
800 'source: 'structural,
801{
802 pub file: &'parsed ProjectFile,
803 pub source: &'source str,
804 pub package_name: &'parsed str,
805 module_fq: &'parsed FqName,
806 pub parsed: &'parsed mut ParsedFile,
807 pub module: Option<CodeUnit>,
808 overload_decorators: PythonOverloadDecoratorBindings,
809 source_facts: PrimarySourceFactCollector<'source>,
810 structural: StructuralFactCollector<'structural, 'structural>,
811 structural_kinds: &'structural CompiledKinds,
812 call_site_context: &'structural CallSiteContext,
813 structural_parents: Vec<Option<u32>>,
814 source_imports: Vec<SourceImportFact>,
815 generic_imports: Vec<SourceImportId>,
816 source_declaration_units: Vec<(SourceDeclarationId, CodeUnit)>,
817 source_declaration_unit_keys: HashSet<CodeUnit>,
820 python_source_facts: PythonSourceFacts,
821 python_callable_return_by_node: HashMap<usize, usize>,
822 overload_metadata: Vec<(CodeUnit, usize, Vec<PythonOverloadDecoratorName>)>,
823}
824
825struct PythonFrame<'tree> {
826 node: Node<'tree>,
827 scope: Vec<Scope>,
828 module_control_depth: usize,
829 declaration: bool,
830 wrapper: Option<Node<'tree>>,
831 lexical_scopes: Vec<Node<'tree>>,
832}
833
834enum PythonWork<'tree> {
835 Enter(PythonFrame<'tree>),
836 Exit,
837}
838
839struct DeclarationChildren<'tree> {
840 body: Option<(Node<'tree>, Vec<Scope>)>,
841 container_scope: Option<Vec<Scope>>,
842 container_depth: usize,
843}
844
845impl<'source, 'parsed, 'structural> PythonVisitor<'source, 'parsed, 'structural>
846where
847 'source: 'structural,
848{
849 pub fn visit_container(
850 &mut self,
851 node: Node<'_>,
852 scope: &[Scope],
853 module_control_depth: usize,
854 ) {
855 let mut stack = vec![PythonWork::Enter(PythonFrame {
856 node,
857 scope: scope.to_vec(),
858 module_control_depth,
859 declaration: true,
860 wrapper: None,
861 lexical_scopes: Vec::new(),
862 })];
863 while let Some(work) = stack.pop() {
864 let PythonWork::Enter(frame) = work else {
865 self.structural_parents
866 .pop()
867 .expect("Python structural parent stack must balance");
868 continue;
869 };
870 let structural_parent = self.structural_parents.last().copied().flatten();
871 let structural_id = self.admit_structural(frame.node, structural_parent);
872 self.structural_parents
873 .push(structural_id.or(structural_parent));
874
875 if frame.node.kind() == "identifier" {
876 let text = py_node_text(frame.node, self.source).trim();
877 if !text.is_empty() {
878 self.parsed.type_identifiers.insert(text.to_string());
879 }
880 }
881
882 if frame.node.kind() == "function_definition" {
883 self.capture_function_return_fact(frame.node, frame.wrapper);
884 }
885
886 if matches!(
887 frame.node.kind(),
888 "import_statement" | "import_from_statement"
889 ) {
890 self.visit_import_statement(frame.node, &frame.lexical_scopes);
891 }
892
893 let declaration_children = if frame.declaration {
894 self.visit_statement(
895 frame.node,
896 &frame.scope,
897 frame.module_control_depth,
898 frame.wrapper,
899 )
900 } else {
901 DeclarationChildren {
902 body: None,
903 container_scope: None,
904 container_depth: frame.module_control_depth,
905 }
906 };
907
908 let mut children = Vec::new();
909 let mut cursor = frame.node.walk();
910 for child in frame.node.named_children(&mut cursor) {
911 let mut child_scope = frame.scope.clone();
912 let mut declaration = false;
913 let mut wrapper = None;
914 let mut child_depth = frame.module_control_depth;
915
916 if let Some((body, scope)) = &declaration_children.body
917 && child.id() == body.id()
918 {
919 declaration = true;
920 child_scope = scope.clone();
921 } else if let Some(scope) = &declaration_children.container_scope {
922 declaration = true;
923 child_scope = scope.clone();
924 child_depth = declaration_children.container_depth;
925 } else if frame.declaration
926 && frame.node.kind() == "decorated_definition"
927 && matches!(child.kind(), "class_definition" | "function_definition")
928 {
929 declaration = true;
932 wrapper = Some(frame.node);
933 }
934
935 let mut lexical_scopes = frame.lexical_scopes.clone();
936 if matches!(
937 frame.node.kind(),
938 "class_definition" | "function_definition" | "lambda"
939 ) {
940 lexical_scopes.push(frame.node);
941 }
942 children.push(PythonFrame {
943 node: child,
944 scope: child_scope,
945 module_control_depth: child_depth,
946 declaration,
947 wrapper,
948 lexical_scopes,
949 });
950 }
951
952 stack.push(PythonWork::Exit);
953 stack.extend(children.into_iter().rev().map(PythonWork::Enter));
954 }
955 }
956
957 fn admit_structural(&mut self, node: Node<'_>, parent: Option<u32>) -> Option<u32> {
958 if !node.is_named() {
959 return parent;
960 }
961 let Some(raw_kind) = self.structural_kinds.kind_of(&node) else {
962 return parent;
963 };
964 if !PYTHON_STRUCTURAL_SPEC.should_extract(node, raw_kind) {
965 return parent;
966 }
967 let kind = PYTHON_STRUCTURAL_SPEC.refine_kind(
968 node,
969 raw_kind,
970 parent.map(|id| self.structural.normalized_kind(id)),
971 self.source,
972 self.call_site_context,
973 );
974 let fact_id = self
975 .structural
976 .enter(node, kind, parent, &mut self.source_facts)
977 .expect("Python structural fact collection must remain unbounded");
978 let mut sink = self.structural.role_sink(&mut self.source_facts);
979 PYTHON_STRUCTURAL_SPEC.extract(node, kind, &mut sink);
980 self.structural
981 .accept_roles(fact_id, sink.into_parts())
982 .expect("Python structural role collection must remain unbounded");
983 Some(fact_id)
984 }
985
986 fn replace_source_declaration(
987 &mut self,
988 declaration_node: Node<'_>,
989 name_node: Option<Node<'_>>,
990 unit: CodeUnit,
991 metadata_ordinal: Option<usize>,
992 ) -> SourceDeclarationId {
993 self.source_declaration_units
994 .retain(|(_, existing)| existing != &unit);
995 self.parsed
996 .source_declaration_metadata
997 .retain(|link| link.unit != unit);
998 self.record_source_declaration(declaration_node, name_node, unit, metadata_ordinal)
999 }
1000
1001 fn record_source_declaration(
1002 &mut self,
1003 declaration_node: Node<'_>,
1004 name_node: Option<Node<'_>>,
1005 unit: CodeUnit,
1006 metadata_ordinal: Option<usize>,
1007 ) -> SourceDeclarationId {
1008 let declaration = self.source_facts.declare_node(declaration_node, name_node);
1009 self.link_source_declaration(declaration, unit, metadata_ordinal);
1010 declaration
1011 }
1012
1013 fn capture_function_return_fact<'tree>(
1014 &mut self,
1015 node: Node<'tree>,
1016 wrapper: Option<Node<'tree>>,
1017 ) {
1018 let declaration_node = wrapper
1019 .or_else(|| {
1020 node.parent()
1021 .filter(|parent| parent.kind() == "decorated_definition")
1022 })
1023 .unwrap_or(node);
1024 let declaration = self
1025 .source_facts
1026 .declare_node(declaration_node, node.child_by_field_name("name"));
1027 let return_annotation = node
1028 .child_by_field_name("return_type")
1029 .map(|annotation| self.source_facts.intern_node(annotation));
1030 let annotation_references = node
1031 .child_by_field_name("return_type")
1032 .map(|annotation| {
1033 python_annotation_references(annotation, self.source, &mut self.source_facts)
1034 })
1035 .unwrap_or_default();
1036 let runtime_type = node
1037 .child_by_field_name("return_type")
1038 .and_then(|annotation| python_return_type_identity(annotation, self.source));
1039 let fact_index = self.python_source_facts.callable_returns.len();
1040 self.python_source_facts
1041 .callable_returns
1042 .push(PythonCallableReturnFact {
1043 declaration,
1044 return_annotation,
1045 annotation_references,
1046 runtime_type,
1047 });
1048 assert!(
1049 self.python_callable_return_by_node
1050 .insert(node.id(), fact_index)
1051 .is_none(),
1052 "each Python function definition must have one callable return fact"
1053 );
1054 }
1055
1056 fn link_source_declaration(
1057 &mut self,
1058 declaration: SourceDeclarationId,
1059 unit: CodeUnit,
1060 metadata_ordinal: Option<usize>,
1061 ) {
1062 self.source_declaration_unit_keys.insert(unit.clone());
1063 self.source_declaration_units
1064 .push((declaration, unit.clone()));
1065 if let Some(metadata_ordinal) = metadata_ordinal {
1066 self.parsed
1067 .source_declaration_metadata
1068 .push(SourceDeclarationMetadataLink {
1069 declaration,
1070 unit,
1071 metadata_ordinal,
1072 });
1073 }
1074 }
1075
1076 fn visit_statement<'tree>(
1077 &mut self,
1078 node: Node<'tree>,
1079 scope: &[Scope],
1080 module_control_depth: usize,
1081 wrapper: Option<Node<'tree>>,
1082 ) -> DeclarationChildren<'tree> {
1083 match node.kind() {
1084 "decorated_definition" => DeclarationChildren {
1085 body: None,
1086 container_scope: None,
1087 container_depth: module_control_depth,
1088 },
1089 "class_definition" | "function_definition" => {
1090 self.visit_definition(node, wrapper, scope, module_control_depth)
1091 }
1092 "expression_statement" => {
1093 self.visit_expression_statement(node, scope, module_control_depth);
1094 DeclarationChildren {
1095 body: None,
1096 container_scope: None,
1097 container_depth: module_control_depth,
1098 }
1099 }
1100 "import_statement" | "import_from_statement" => DeclarationChildren {
1101 body: None,
1102 container_scope: None,
1103 container_depth: module_control_depth,
1104 },
1105 "if_statement" | "try_statement" | "with_statement" | "for_statement"
1106 | "while_statement" => {
1107 let next_depth = if scope.is_empty() {
1108 module_control_depth + 1
1109 } else {
1110 module_control_depth
1111 };
1112 DeclarationChildren {
1113 body: None,
1114 container_scope: Some(scope.to_vec()),
1115 container_depth: next_depth,
1116 }
1117 }
1118 "elif_clause" | "else_clause" | "except_clause" | "finally_clause" => {
1119 DeclarationChildren {
1120 body: None,
1121 container_scope: Some(scope.to_vec()),
1122 container_depth: module_control_depth,
1123 }
1124 }
1125 "block" | "module" => DeclarationChildren {
1126 body: None,
1127 container_scope: Some(scope.to_vec()),
1128 container_depth: module_control_depth,
1129 },
1130 _ => DeclarationChildren {
1131 body: None,
1132 container_scope: None,
1133 container_depth: module_control_depth,
1134 },
1135 }
1136 }
1137
1138 fn visit_definition<'tree>(
1139 &mut self,
1140 definition: Node<'tree>,
1141 wrapper: Option<Node<'tree>>,
1142 scope: &[Scope],
1143 module_control_depth: usize,
1144 ) -> DeclarationChildren<'tree> {
1145 match definition.kind() {
1146 "class_definition" => self.visit_class_definition(
1147 definition,
1148 wrapper.unwrap_or(definition),
1149 scope,
1150 module_control_depth,
1151 ),
1152 "function_definition" => self.visit_function_definition(
1153 definition,
1154 wrapper.unwrap_or(definition),
1155 scope,
1156 module_control_depth,
1157 ),
1158 _ => DeclarationChildren {
1159 body: None,
1160 container_scope: None,
1161 container_depth: module_control_depth,
1162 },
1163 }
1164 }
1165
1166 fn visit_class_definition<'tree>(
1167 &mut self,
1168 node: Node<'tree>,
1169 range_node: Node<'tree>,
1170 scope: &[Scope],
1171 module_control_depth: usize,
1172 ) -> DeclarationChildren<'tree> {
1173 let Some(name_node) = node.child_by_field_name("name") else {
1174 return DeclarationChildren {
1175 body: None,
1176 container_scope: None,
1177 container_depth: module_control_depth,
1178 };
1179 };
1180 let name = py_node_text(name_node, self.source).trim();
1181 if name.is_empty() {
1182 return DeclarationChildren {
1183 body: None,
1184 container_scope: None,
1185 container_depth: module_control_depth,
1186 };
1187 }
1188
1189 let capture = !scope.is_empty() || module_control_depth <= 1;
1190
1191 let short_name = scope
1192 .last()
1193 .map(|parent| format!("{}${name}", parent.path))
1194 .unwrap_or_else(|| name.to_string());
1195 let fq = match scope.last() {
1201 Some(parent) => parent
1202 .fq
1203 .clone()
1204 .with_pushed(segment_interner().intern(name, SegmentKind::Nested)),
1205 None => self
1206 .module_fq
1207 .clone()
1208 .with_pushed(segment_interner().intern(name, SegmentKind::Type)),
1209 };
1210 let code_unit = CodeUnit::new_fq(
1211 self.file.clone(),
1212 CodeUnitType::Class,
1213 self.package_name.to_string(),
1214 short_name.clone(),
1215 fq.clone(),
1216 );
1217 if capture {
1218 self.parsed
1219 .replace_code_unit(code_unit.clone(), range_node, self.source, None, None);
1220 self.parsed.add_signature(
1221 code_unit.clone(),
1222 python_class_signature(range_node, self.source),
1223 );
1224 if let Some(module) = &self.module
1225 && scope.is_empty()
1226 {
1227 self.parsed.add_child(module.clone(), code_unit.clone());
1228 }
1229 if let Some(parent) = scope.last()
1230 && let Some(parent_cu) = &parent.code_unit
1231 {
1232 self.parsed.add_child(parent_cu.clone(), code_unit.clone());
1233 }
1234 self.parsed.set_raw_supertypes(
1235 code_unit.clone(),
1236 extract_python_supertypes(node, self.source),
1237 );
1238 self.replace_source_declaration(range_node, Some(name_node), code_unit.clone(), None);
1239 }
1240
1241 let mut next_scope = scope.to_vec();
1242 if capture {
1243 next_scope.push(Scope {
1244 kind: ScopeKind::Class,
1245 path: short_name,
1246 fq,
1247 code_unit: Some(code_unit),
1248 method_receiver: None,
1249 });
1250 }
1251 DeclarationChildren {
1252 body: node
1253 .child_by_field_name("body")
1254 .map(|body| (body, next_scope)),
1255 container_scope: None,
1256 container_depth: module_control_depth,
1257 }
1258 }
1259
1260 fn visit_function_definition<'tree>(
1261 &mut self,
1262 node: Node<'tree>,
1263 range_node: Node<'tree>,
1264 scope: &[Scope],
1265 module_control_depth: usize,
1266 ) -> DeclarationChildren<'tree> {
1267 let Some(name_node) = node.child_by_field_name("name") else {
1268 return DeclarationChildren {
1269 body: None,
1270 container_scope: None,
1271 container_depth: module_control_depth,
1272 };
1273 };
1274 let name = py_node_text(name_node, self.source).trim();
1275 if name.is_empty() {
1276 return DeclarationChildren {
1277 body: None,
1278 container_scope: None,
1279 container_depth: module_control_depth,
1280 };
1281 }
1282
1283 let capture = !python_is_property_mutator(range_node, self.source)
1291 && ((scope.is_empty() && module_control_depth <= 1)
1292 || scope
1293 .last()
1294 .is_some_and(|parent| parent.kind == ScopeKind::Class));
1295 let short_name = if let Some(parent) = scope.last() {
1296 match parent.kind {
1297 ScopeKind::Class => format!("{}.{}", parent.path, name),
1298 ScopeKind::Function => format!("{}${name}", parent.path),
1299 }
1300 } else {
1301 name.to_string()
1302 };
1303 let fq = if let Some(parent) = scope.last() {
1308 match parent.kind {
1309 ScopeKind::Class => parent
1310 .fq
1311 .clone()
1312 .with_pushed(segment_interner().intern(name, SegmentKind::Member)),
1313 ScopeKind::Function => parent
1314 .fq
1315 .clone()
1316 .with_pushed(segment_interner().intern(name, SegmentKind::Nested)),
1317 }
1318 } else {
1319 self.module_fq
1320 .clone()
1321 .with_pushed(segment_interner().intern(name, SegmentKind::Member))
1322 };
1323
1324 if capture {
1325 let code_unit_type = if python_function_has_property_decorator(node, self.source) {
1326 CodeUnitType::Field
1327 } else {
1328 CodeUnitType::Function
1329 };
1330 let signature = node
1331 .child_by_field_name("parameters")
1332 .map(|parameters| py_node_text(parameters, self.source).trim().to_string());
1333 let code_unit = CodeUnit::with_signature_and_fq(
1334 self.file.clone(),
1335 code_unit_type,
1336 self.package_name.to_string(),
1337 short_name.clone(),
1338 signature,
1339 false,
1340 fq.clone(),
1341 );
1342 let replaces_declaration = self.parsed.contains_declaration(&code_unit);
1345 let replaces_links = self.source_declaration_unit_keys.contains(&code_unit);
1346 if replaces_declaration {
1347 self.parsed.replace_code_unit(
1348 code_unit.clone(),
1349 range_node,
1350 self.source,
1351 None,
1352 None,
1353 );
1354 } else {
1355 self.parsed
1356 .add_code_unit(code_unit.clone(), range_node, self.source, None, None);
1357 }
1358 let signature = python_function_signature(node, self.source);
1359 let fact_index = *self
1360 .python_callable_return_by_node
1361 .get(&node.id())
1362 .expect("function return fact must precede declaration admission");
1363 let return_fact = &self.python_source_facts.callable_returns[fact_index];
1364 let return_declaration = return_fact.declaration;
1365 let return_type_identity = return_fact.runtime_type.clone();
1366 let return_type_text = return_fact.return_annotation.map(|annotation| {
1367 let range = self.source_facts.occurrence(annotation).range;
1368 self.source[range.start_byte..range.end_byte].to_string()
1369 });
1370 let metadata_ordinal = self.parsed.add_signature_with_metadata(
1371 code_unit.clone(),
1372 python_signature_metadata(
1373 signature,
1374 node,
1375 self.source,
1376 return_type_text.as_deref(),
1377 return_type_identity,
1378 ),
1379 );
1380 if replaces_links {
1381 self.overload_metadata
1382 .retain(|(existing, _, _)| existing != &code_unit);
1383 self.source_declaration_units
1384 .retain(|(_, existing)| existing != &code_unit);
1385 self.parsed
1386 .source_declaration_metadata
1387 .retain(|link| link.unit != code_unit);
1388 }
1389 self.overload_metadata.push((
1390 code_unit.clone(),
1391 metadata_ordinal,
1392 PythonOverloadDecoratorBindings::overload_decorator_names(node, self.source),
1393 ));
1394 self.link_source_declaration(
1395 return_declaration,
1396 code_unit.clone(),
1397 Some(metadata_ordinal),
1398 );
1399 if let Some(module) = &self.module
1400 && scope.is_empty()
1401 {
1402 self.parsed.add_child(module.clone(), code_unit.clone());
1403 }
1404 if let Some(parent) = scope.last()
1405 && parent.kind == ScopeKind::Class
1406 && let Some(parent_cu) = &parent.code_unit
1407 {
1408 self.parsed.add_child(parent_cu.clone(), code_unit.clone());
1409 }
1410 let scope_code_unit = Some(code_unit);
1411 let mut next_scope = scope.to_vec();
1412 next_scope.push(Scope {
1413 kind: ScopeKind::Function,
1414 path: short_name,
1415 fq,
1416 code_unit: scope_code_unit,
1417 method_receiver: scope
1418 .last()
1419 .is_some_and(|parent| parent.kind == ScopeKind::Class)
1420 .then(|| python_instance_method_receiver_name(node, self.source))
1421 .flatten(),
1422 });
1423 return DeclarationChildren {
1424 body: node
1425 .child_by_field_name("body")
1426 .map(|body| (body, next_scope)),
1427 container_scope: None,
1428 container_depth: module_control_depth,
1429 };
1430 }
1431
1432 let mut next_scope = scope.to_vec();
1433 next_scope.push(Scope {
1434 kind: ScopeKind::Function,
1435 path: short_name,
1436 fq,
1437 code_unit: None,
1438 method_receiver: None,
1439 });
1440 DeclarationChildren {
1441 body: node
1442 .child_by_field_name("body")
1443 .map(|body| (body, next_scope)),
1444 container_scope: None,
1445 container_depth: module_control_depth,
1446 }
1447 }
1448
1449 fn visit_expression_statement(
1450 &mut self,
1451 node: Node<'_>,
1452 scope: &[Scope],
1453 module_control_depth: usize,
1454 ) {
1455 let Some(assignment) = node.named_child(0) else {
1456 return;
1457 };
1458 if assignment.kind() != "assignment" {
1459 return;
1460 }
1461 let targets = python_chained_assignment_targets(assignment);
1462 if targets.is_empty() {
1463 return;
1464 }
1465 for left in &targets {
1466 self.visit_instance_attribute_assignment(*left, scope);
1467 }
1468 let names = targets
1469 .iter()
1470 .flat_map(|left| collect_assigned_name_nodes(*left, self.source));
1471 for name_node in names {
1472 let name = py_node_text(name_node, self.source).trim().to_string();
1473 if name.is_empty() {
1474 continue;
1475 }
1476 let (short_name, fq) = if let Some(parent) = scope.last() {
1477 if parent.kind != ScopeKind::Class {
1478 continue;
1479 }
1480 (
1481 format!("{}.{}", parent.path, name),
1482 parent
1483 .fq
1484 .clone()
1485 .with_pushed(segment_interner().intern(&name, SegmentKind::Member)),
1486 )
1487 } else if module_control_depth <= 1 {
1488 (
1489 name.clone(),
1490 self.module_fq
1491 .clone()
1492 .with_pushed(segment_interner().intern(&name, SegmentKind::Member)),
1493 )
1494 } else {
1495 continue;
1496 };
1497 let code_unit = CodeUnit::new_fq(
1498 self.file.clone(),
1499 CodeUnitType::Field,
1500 self.package_name.to_string(),
1501 short_name,
1502 fq,
1503 );
1504 self.parsed
1510 .add_code_unit(code_unit.clone(), node, self.source, None, None);
1511 self.parsed.add_signature(
1512 code_unit.clone(),
1513 py_node_text(node, self.source).trim().to_string(),
1514 );
1515 self.record_source_declaration(node, Some(name_node), code_unit.clone(), None);
1516 if let Some(module) = &self.module
1517 && scope.is_empty()
1518 {
1519 self.parsed.add_child(module.clone(), code_unit.clone());
1520 }
1521 if let Some(parent) = scope.last()
1522 && parent.kind == ScopeKind::Class
1523 && let Some(parent_cu) = &parent.code_unit
1524 {
1525 self.parsed.add_child(parent_cu.clone(), code_unit);
1526 }
1527 }
1528 }
1529
1530 fn visit_instance_attribute_assignment(&mut self, left: Node<'_>, scope: &[Scope]) {
1531 let Some(function) = scope
1532 .last()
1533 .filter(|scope| scope.kind == ScopeKind::Function)
1534 else {
1535 return;
1536 };
1537 let Some(receiver) = function.method_receiver.as_deref() else {
1538 return;
1539 };
1540 let Some(parent) = scope
1541 .get(scope.len().saturating_sub(2))
1542 .filter(|scope| scope.kind == ScopeKind::Class)
1543 else {
1544 return;
1545 };
1546 let Some(parent_cu) = parent.code_unit.clone() else {
1547 return;
1548 };
1549 for (name, node) in collect_self_assigned_attributes(left, self.source, receiver) {
1550 let code_unit = CodeUnit::new_fq(
1551 self.file.clone(),
1552 CodeUnitType::Field,
1553 self.package_name.to_string(),
1554 format!("{}.{}", parent.path, name),
1555 parent
1556 .fq
1557 .clone()
1558 .with_pushed(segment_interner().intern(&name, SegmentKind::Member)),
1559 );
1560 if !self.parsed.contains_declaration(&code_unit) {
1561 self.parsed.replace_code_unit(
1562 code_unit.clone(),
1563 node,
1564 self.source,
1565 Some(parent_cu.clone()),
1566 Some(parent_cu.clone()),
1567 );
1568 self.record_source_declaration(node, Some(node), code_unit.clone(), None);
1569 }
1570 self.parsed.add_signature(
1571 code_unit.clone(),
1572 py_node_text(node, self.source).trim().to_string(),
1573 );
1574 }
1575 }
1576
1577 fn visit_import_statement(&mut self, node: Node<'_>, lexical_scopes: &[Node<'_>]) {
1578 for syntax in python_import_syntaxes_from_node(node, self.source) {
1579 if lexical_scopes.is_empty() {
1580 self.overload_decorators.collect_import(&syntax.info);
1581 }
1582 let declaration = self.source_facts.intern_node(syntax.declaration);
1583 let target = syntax
1584 .target
1585 .map(|node| self.source_facts.intern_node(node));
1586 let alias_occurrence = syntax.alias.map(|node| self.source_facts.intern_node(node));
1587 let lexical_scopes = lexical_scopes
1588 .iter()
1589 .map(|node| self.source_facts.intern_node(*node))
1590 .collect();
1591 let source_import = SourceImportFact::from_import(
1592 syntax.info,
1593 declaration,
1594 target,
1595 alias_occurrence,
1596 lexical_scopes,
1597 );
1598 let id = SourceImportId::try_from_index(self.source_imports.len())
1599 .expect("Python source import ids must fit in a u32");
1600 self.source_imports.push(source_import);
1601 self.generic_imports.push(id);
1602 }
1603 }
1604
1605 fn finish(
1606 self,
1607 root: Node<'_>,
1608 native_step: impl FnOnce(
1609 &mut ParsedFile,
1610 &mut PrimarySourceFactCollector<'source>,
1611 &[(SourceDeclarationId, CodeUnit)],
1612 ) -> NativeSourceBridges,
1613 ) {
1614 let PythonVisitor {
1615 parsed,
1616 source,
1617 mut source_facts,
1618 structural,
1619 structural_parents,
1620 source_imports,
1621 generic_imports,
1622 source_declaration_units,
1623 python_source_facts,
1624 python_callable_return_by_node: _,
1625 overload_decorators,
1626 overload_metadata,
1627 ..
1628 } = self;
1629 assert_eq!(structural_parents, vec![None]);
1630 let structural = structural
1631 .finish(root)
1632 .expect("Python structural fact collection must finish");
1633 let (native_site_occurrences, native_declaration_sources) =
1634 native_step(parsed, &mut source_facts, &source_declaration_units);
1635 let occurrences = source_facts.finish();
1636 assert!(
1637 python_source_facts.valid_links(&occurrences),
1638 "Python callable return facts must link to exact source identities"
1639 );
1640 for (unit, metadata_ordinal, decorators) in overload_metadata {
1641 let declaration_only = decorators
1642 .iter()
1643 .any(|name| overload_decorators.matches_overload_binding(name));
1644 if declaration_only
1645 && let Some(metadata) = parsed
1646 .signature_metadata
1647 .get_mut(&unit)
1648 .and_then(|entries| entries.get_mut(metadata_ordinal))
1649 {
1650 *metadata = metadata.clone().with_declaration_only(true);
1651 }
1652 }
1653 parsed.imports = generic_imports
1654 .iter()
1655 .map(|id| source_imports[id.index()].import_info(&occurrences))
1656 .collect();
1657 parsed.source_declaration_units = source_declaration_units;
1658 parsed.source_facts = Some(ParsedSourceFacts {
1659 cpp: None,
1660 go: None,
1661 java: None,
1662 js_ts: None,
1663 php: None,
1664 scala: None,
1665 ruby: None,
1666 source_bytes: source.len(),
1667 occurrences,
1668 structural,
1669 python: Some(python_source_facts),
1670 native_site_occurrences,
1671 native_declaration_sources,
1672 declaration_visibilities: None,
1673 rust_declaration_properties: Vec::new(),
1674 rust_modules: None,
1675 rust_types: Vec::new(),
1676 rust_items: RustItemSourceFacts::default(),
1677 imports: source_imports,
1678 generic_imports,
1679 rust_import_contexts: Vec::new(),
1680 });
1681 }
1682}
1683
1684type NativeSourceBridges = (
1685 Vec<SourceOccurrenceId>,
1686 Vec<(ResolutionSiteId, SourceDeclarationId)>,
1687);
1688
1689pub fn parse_python_file(
1692 project: &dyn Project,
1693 file: &ProjectFile,
1694 source: &str,
1695 tree: &Tree,
1696) -> ParsedFile {
1697 parse_python_file_with_step(project, file, source, tree, |_, _, _| Default::default())
1698}
1699
1700#[cfg(any(test, feature = "test-support"))]
1703pub fn parse_python_file_with_native_facts(
1704 project: &dyn Project,
1705 file: &ProjectFile,
1706 source: &str,
1707 tree: &Tree,
1708) -> ParsedFile {
1709 parse_python_file_with_step(project, file, source, tree, |parsed, collector, units| {
1710 let (facts, occurrences, declarations) = crate::resolution::produce(
1713 tree.root_node(),
1714 source,
1715 collector,
1716 units
1717 .iter()
1718 .filter(|(_, unit)| parsed.contains_declaration(unit)),
1719 );
1720 parsed.resolution_facts = facts;
1721 (occurrences, declarations)
1722 })
1723}
1724
1725fn parse_python_file_with_step<'source>(
1726 project: &dyn Project,
1727 file: &ProjectFile,
1728 source: &'source str,
1729 tree: &Tree,
1730 native_step: impl FnOnce(
1731 &mut ParsedFile,
1732 &mut PrimarySourceFactCollector<'source>,
1733 &[(SourceDeclarationId, CodeUnit)],
1734 ) -> NativeSourceBridges,
1735) -> ParsedFile {
1736 let module_components = python_module_components(project, file);
1737 let module_name = module_components.join(".");
1738 let module_fq = FqName::from_package_components(&module_components);
1739 let mut parsed = ParsedFile::new(module_name.clone());
1740 let root = tree.root_node();
1741
1742 let module_code_unit = module_code_unit_from_fq(file, &module_components, module_fq.clone());
1743 if let Some(module) = module_code_unit.clone() {
1744 parsed.add_code_unit(module, root, source, None, None);
1745 }
1746
1747 let overload_decorators = PythonOverloadDecoratorBindings::default();
1748 let structural_kinds = CompiledKinds::compile(
1749 &tree_sitter_python::LANGUAGE.into(),
1750 PYTHON_STRUCTURAL_SPEC.kind_table(),
1751 );
1752 let call_site_context = PYTHON_STRUCTURAL_SPEC.call_site_context(root, source);
1753 let mut visitor = PythonVisitor {
1754 file,
1755 source,
1756 package_name: &module_name,
1757 module_fq: &module_fq,
1758 parsed: &mut parsed,
1759 module: module_code_unit,
1760 overload_decorators,
1761 source_facts: PrimarySourceFactCollector::new(source),
1762 structural: StructuralFactCollector::new(
1763 &PYTHON_STRUCTURAL_SPEC,
1764 source,
1765 &call_site_context,
1766 ParentIndex::new(root),
1767 usize::MAX,
1768 None,
1769 ),
1770 structural_kinds: &structural_kinds,
1771 call_site_context: &call_site_context,
1772 structural_parents: vec![None],
1773 source_imports: Vec::new(),
1774 generic_imports: Vec::new(),
1775 source_declaration_units: Vec::new(),
1776 source_declaration_unit_keys: HashSet::default(),
1777 python_source_facts: PythonSourceFacts::default(),
1778 python_callable_return_by_node: HashMap::default(),
1779 overload_metadata: Vec::new(),
1780 };
1781 visitor.visit_container(root, &[], 0);
1782 visitor.finish(root, native_step);
1783
1784 parsed
1785}
1786
1787pub use brokk_bifrost_core::analyzer::common::node_source_text as py_node_text;
1788
1789pub fn python_module_name(project: &dyn Project, file: &ProjectFile) -> String {
1790 python_module_components(project, file).join(".")
1791}
1792
1793pub fn module_code_unit_from_name(file: &ProjectFile, name: &str) -> Option<CodeUnit> {
1795 let components = python_module_components_from_name(file, name)?;
1796 let fq = FqName::from_package_components(&components);
1797 module_code_unit_from_fq(file, &components, fq)
1798}
1799
1800pub fn module_code_unit(project: &dyn Project, file: &ProjectFile) -> Option<CodeUnit> {
1801 let components = python_module_components(project, file);
1802 let structured_fq = FqName::from_package_components(&components);
1803 module_code_unit_from_fq(file, &components, structured_fq)
1804}
1805
1806fn module_code_unit_from_fq(
1807 file: &ProjectFile,
1808 components: &[String],
1809 structured_fq: FqName,
1810) -> Option<CodeUnit> {
1811 let (short_name, package_components) = components.split_last()?;
1812 let package_name = package_components.join(".");
1813 Some(CodeUnit::new_fq(
1814 file.clone(),
1815 CodeUnitType::Module,
1816 package_name,
1817 short_name.clone(),
1818 structured_fq,
1819 ))
1820}
1821
1822fn python_class_signature(node: Node<'_>, source: &str) -> String {
1823 python_header_with_decorators(node, source)
1824}
1825
1826fn python_function_signature(function: Node<'_>, source: &str) -> String {
1827 let header_end = (0..function.child_count())
1828 .rev()
1829 .filter_map(|index| function.child(index))
1830 .find(|child| child.kind() == ":")
1831 .map_or_else(
1832 || {
1833 function
1834 .child_by_field_name("body")
1835 .map_or(function.end_byte(), |body| body.start_byte())
1836 },
1837 |colon| colon.end_byte(),
1838 );
1839 let header_node = python_function_header_node(function);
1840 let header_start = header_node.start_byte();
1841 let header = source
1842 .get(header_start..header_end)
1843 .expect("Python function header must be a source range")
1844 .trim_end();
1845 format!(
1846 "{} ...",
1847 python_dedent_function_header(header, header_node.start_position().column)
1848 )
1849}
1850
1851fn python_dedent_function_header(header: &str, base_indent: usize) -> String {
1852 let mut lines = header.lines();
1853 let Some(first) = lines.next() else {
1854 return String::new();
1855 };
1856 let mut dedented = String::with_capacity(header.len());
1857 dedented.push_str(first);
1858 for line in lines {
1859 dedented.push('\n');
1860 dedented.push_str(&line[python_header_indent_bytes(line, base_indent)..]);
1861 }
1862 dedented
1863}
1864
1865fn python_dedented_header_offset(source_prefix: &str, base_indent: usize) -> usize {
1868 let mut removed_bytes = 0;
1869 let mut line_start = 0;
1870 let mut continuation_line = false;
1871
1872 while line_start < source_prefix.len() {
1873 let newline = source_prefix.as_bytes()[line_start..]
1874 .iter()
1875 .position(|byte| *byte == b'\n')
1876 .map(|offset| line_start + offset);
1877 let line_end = newline.unwrap_or(source_prefix.len());
1878 let line = &source_prefix[line_start..line_end];
1879 if continuation_line {
1880 removed_bytes += python_header_indent_bytes(line, base_indent);
1881 }
1882 let Some(newline) = newline else {
1883 break;
1884 };
1885 if newline > 0 && source_prefix.as_bytes()[newline - 1] == b'\r' {
1886 removed_bytes += 1;
1887 }
1888 line_start = newline + 1;
1889 continuation_line = true;
1890 }
1891
1892 source_prefix.len() - removed_bytes
1893}
1894
1895fn python_header_indent_bytes(line: &str, base_indent: usize) -> usize {
1896 line.bytes()
1897 .take_while(|byte| matches!(*byte, b' ' | b'\t'))
1898 .count()
1899 .min(base_indent)
1900}
1901
1902fn python_signature_metadata(
1903 signature: String,
1904 node: Node<'_>,
1905 source: &str,
1906 return_type_text: Option<&str>,
1907 return_type_identity: Option<StructuredTypeIdentity>,
1908) -> SignatureMetadata {
1909 let Some(parameters_node) = node.child_by_field_name("parameters") else {
1910 return SignatureMetadata::new(signature, Vec::new())
1911 .with_return_type_text(return_type_text)
1912 .with_return_type_identity(return_type_identity)
1913 .with_dispatch_extensibility(DispatchExtensibility::Open)
1914 .with_callable_modifiers(
1915 python_callable_is_static(node, source),
1916 false,
1917 DeclaredVisibility::Unknown,
1918 );
1919 };
1920 let arity = python_callable_arity(parameters_node, source);
1921 let header_node = python_function_header_node(node);
1926 let header_start = header_node.start_byte();
1927 let base_indent = header_node.start_position().column;
1928 let parameters = python_parameter_label_nodes(parameters_node)
1929 .into_iter()
1930 .map(|label_node| {
1931 let label = py_node_text(label_node, source);
1932 let label_range = label_node.byte_range();
1933 let source_prefix = source
1934 .get(header_start..label_range.start)
1935 .expect("Python parameter must follow its function header");
1936 let start_byte = python_dedented_header_offset(source_prefix, base_indent);
1937 let end_byte = start_byte + label.len();
1938 assert_eq!(
1939 signature.get(start_byte..end_byte),
1940 Some(label),
1941 "Python parameter metadata must point at its exact signature label"
1942 );
1943 ParameterMetadata::new(label, start_byte, end_byte)
1944 })
1945 .collect();
1946 let metadata = SignatureMetadata::new(signature, parameters)
1947 .with_return_type_text(return_type_text)
1948 .with_return_type_identity(return_type_identity)
1949 .with_dispatch_extensibility(DispatchExtensibility::Open)
1950 .with_callable_modifiers(
1951 python_callable_is_static(node, source),
1952 false,
1953 DeclaredVisibility::Unknown,
1954 );
1955 if let Some(arity) = arity {
1956 metadata.with_callable_arity(arity)
1957 } else {
1958 metadata
1959 }
1960}
1961
1962fn python_callable_arity(parameters: Node<'_>, source: &str) -> Option<CallableArity> {
1967 let mut required = 0;
1968 let mut total = 0;
1969 let mut repeated = false;
1970 let mut keyword_only = false;
1971 let mut cursor = parameters.walk();
1972 for parameter in parameters.named_children(&mut cursor) {
1973 match parameter.kind() {
1974 "positional_separator" => continue,
1975 "keyword_separator" => {
1976 keyword_only = true;
1977 continue;
1978 }
1979 "ERROR" => return None,
1980 _ => {}
1981 }
1982 if !matches!(
1983 parameter.kind(),
1984 "identifier"
1985 | "typed_parameter"
1986 | "default_parameter"
1987 | "typed_default_parameter"
1988 | "list_splat_pattern"
1989 | "dictionary_splat_pattern"
1990 ) {
1991 continue;
1992 }
1993 let label = python_parameter_label_node(parameter)?;
1994 source.get(label.byte_range())?;
1995 match python_parameter_splat(parameter) {
1996 Some(PythonParameterSplat::Positional) => {
1997 repeated = true;
1998 keyword_only = true;
1999 }
2000 Some(PythonParameterSplat::Keyword) => {}
2001 None if !keyword_only => {
2002 total += 1;
2003 if !matches!(
2004 parameter.kind(),
2005 "default_parameter" | "typed_default_parameter"
2006 ) {
2007 required += 1;
2008 }
2009 }
2010 None => {}
2011 }
2012 }
2013 Some(CallableArity::new(required, total, repeated))
2014}
2015
2016fn python_function_header_node(node: Node<'_>) -> Node<'_> {
2017 node.parent()
2018 .filter(|parent| parent.kind() == "decorated_definition")
2019 .unwrap_or(node)
2020}
2021
2022fn python_callable_is_static(node: Node<'_>, source: &str) -> bool {
2047 debug_assert!(
2048 node.kind() == "function_definition",
2049 "Python callable signature metadata is built for function_definition, not {}",
2050 node.kind()
2051 );
2052 python_function_has_decorator(node, source, "staticmethod")
2053 || python_function_has_decorator(node, source, "classmethod")
2054}
2055
2056fn python_parameter_label_nodes(parameters_node: Node<'_>) -> Vec<Node<'_>> {
2057 brokk_bifrost_core::analyzer::tree_walk::named_children_iter(parameters_node)
2058 .filter_map(python_parameter_label_node)
2059 .collect()
2060}
2061
2062#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2079pub enum PythonParameterSplat {
2080 Positional,
2082 Keyword,
2084}
2085
2086pub fn python_parameter_splat(parameter: Node<'_>) -> Option<PythonParameterSplat> {
2087 let splat = match parameter.kind() {
2088 kind @ ("list_splat_pattern" | "dictionary_splat_pattern") => kind,
2089 _ => {
2090 let mut cursor = parameter.walk();
2091 parameter
2092 .named_children(&mut cursor)
2093 .map(|child| child.kind())
2094 .find(|kind| matches!(*kind, "list_splat_pattern" | "dictionary_splat_pattern"))?
2095 }
2096 };
2097 match splat {
2098 "list_splat_pattern" => Some(PythonParameterSplat::Positional),
2099 "dictionary_splat_pattern" => Some(PythonParameterSplat::Keyword),
2100 _ => unreachable!("the splat kind was matched above"),
2101 }
2102}
2103
2104pub fn python_parameter_label_node(node: Node<'_>) -> Option<Node<'_>> {
2105 match node.kind() {
2106 "identifier" => Some(node),
2107 "typed_parameter"
2108 | "typed_default_parameter"
2109 | "default_parameter"
2110 | "list_splat_pattern"
2111 | "dictionary_splat_pattern"
2112 | "keyword_separator" => node.child_by_field_name("name").or_else(|| {
2113 let mut cursor = node.walk();
2114 node.named_children(&mut cursor)
2115 .find_map(python_parameter_label_node)
2116 }),
2117 _ => None,
2118 }
2119}
2120
2121fn python_is_property_mutator(node: Node<'_>, source: &str) -> bool {
2122 python_header_with_decorators(node, source)
2123 .lines()
2124 .map(str::trim)
2125 .filter(|line| line.starts_with('@'))
2126 .any(|decorator| decorator.ends_with(".setter") || decorator.ends_with(".deleter"))
2127}
2128
2129pub fn python_expanded_comment_start(source: &str, start_byte: usize) -> usize {
2130 let line_starts = compute_line_starts(source);
2131 let line_index = find_line_index_for_offset(&line_starts, start_byte);
2132
2133 let mut comment_start = start_byte;
2134 for line_idx in (0..line_index).rev() {
2135 let line_start = line_starts[line_idx];
2136 let line_end = line_starts
2137 .get(line_idx + 1)
2138 .copied()
2139 .unwrap_or(source.len());
2140 let line = &source[line_start..line_end];
2141 let trimmed = line.trim_start();
2142
2143 if trimmed.trim().is_empty() {
2144 continue;
2145 }
2146
2147 if trimmed.starts_with('#') {
2148 comment_start = line_start;
2149 continue;
2150 }
2151
2152 break;
2153 }
2154
2155 comment_start
2156}
2157
2158fn python_header_with_decorators(node: Node<'_>, source: &str) -> String {
2159 let raw = py_node_text(node, source);
2160 let lines: Vec<_> = raw
2161 .lines()
2162 .map(str::trim_end)
2163 .filter(|line| !line.trim().is_empty())
2164 .collect();
2165 let mut relevant = Vec::new();
2166 for line in lines {
2167 let trimmed = line.trim_start();
2168 if trimmed.starts_with('@')
2169 || trimmed.starts_with("def ")
2170 || trimmed.starts_with("async def ")
2171 || trimmed.starts_with("class ")
2172 {
2173 relevant.push(trimmed.to_string());
2174 if trimmed.starts_with("def ")
2175 || trimmed.starts_with("async def ")
2176 || trimmed.starts_with("class ")
2177 {
2178 break;
2179 }
2180 }
2181 }
2182 relevant.join("\n")
2183}
2184
2185fn extract_python_supertypes(node: Node<'_>, source: &str) -> Vec<String> {
2204 let Some(superclasses) = node.child_by_field_name("superclasses") else {
2205 return Vec::new();
2206 };
2207 let mut result = Vec::new();
2208 let mut cursor = superclasses.walk();
2209 for child in superclasses.named_children(&mut cursor) {
2210 if child.kind() == "keyword_argument" {
2211 continue;
2212 }
2213 let named = python_base_origin_node(child);
2214 let text = py_node_text(named, source).trim();
2215 if !text.is_empty() {
2216 result.push(text.to_string());
2217 }
2218 }
2219 result
2220}
2221
2222pub fn python_base_origin_node<'tree>(base: Node<'tree>) -> Node<'tree> {
2225 if base.kind() != "subscript" {
2226 return base;
2227 }
2228 let Some(value) = base.child_by_field_name("value") else {
2229 return base;
2230 };
2231 if matches!(value.kind(), "identifier" | "attribute") {
2232 value
2233 } else {
2234 base
2235 }
2236}
2237
2238fn collect_assigned_name_nodes<'tree>(node: Node<'tree>, _source: &str) -> Vec<Node<'tree>> {
2239 let mut names = Vec::new();
2240 walk_named_tree_preorder(node, true, |node| {
2241 match node.kind() {
2242 "attribute" | "subscript" => WalkControl::SkipChildren,
2246 "identifier" => {
2247 names.push(node);
2248 WalkControl::Continue
2249 }
2250 _ => WalkControl::Continue,
2251 }
2252 });
2253 names
2254}
2255
2256fn collect_self_assigned_attributes<'tree>(
2257 node: Node<'tree>,
2258 source: &str,
2259 receiver_name: &str,
2260) -> Vec<(String, Node<'tree>)> {
2261 let mut attributes = Vec::new();
2262 collect_direct_self_assigned_attributes(node, source, receiver_name, &mut attributes);
2263 attributes
2264}
2265
2266fn collect_direct_self_assigned_attributes<'tree>(
2267 node: Node<'tree>,
2268 source: &str,
2269 receiver_name: &str,
2270 attributes: &mut Vec<(String, Node<'tree>)>,
2271) {
2272 let mut pending = vec![node];
2273 while let Some(node) = pending.pop() {
2274 match node.kind() {
2275 "attribute" => {
2276 let Some(object) = node.child_by_field_name("object") else {
2277 continue;
2278 };
2279 if object.kind() != "identifier"
2280 || py_node_text(object, source).trim() != receiver_name
2281 {
2282 continue;
2283 }
2284 let Some(attribute) = node.child_by_field_name("attribute") else {
2285 continue;
2286 };
2287 let name = py_node_text(attribute, source).trim();
2288 if !name.is_empty() {
2289 attributes.push((name.to_string(), attribute));
2290 }
2291 }
2292 "pattern_list"
2297 | "tuple_pattern"
2298 | "list_pattern"
2299 | "tuple"
2300 | "list"
2301 | "parenthesized_expression" => {
2302 let mut cursor = node.walk();
2303 let children: Vec<_> = node.named_children(&mut cursor).collect();
2304 pending.extend(children.into_iter().rev());
2305 }
2306 _ => {}
2307 }
2308 }
2309}
2310
2311fn python_instance_method_receiver_name(node: Node<'_>, source: &str) -> Option<String> {
2312 if python_function_has_decorator(node, source, "staticmethod")
2313 || python_function_has_decorator(node, source, "classmethod")
2314 {
2315 return None;
2316 }
2317 python_first_parameter_name(node, source)
2318}
2319
2320pub fn python_function_has_decorator(node: Node<'_>, source: &str, decorator_name: &str) -> bool {
2321 let Some(parent) = node.parent() else {
2322 return false;
2323 };
2324 if parent.kind() != "decorated_definition" {
2325 return false;
2326 }
2327 let mut cursor = parent.walk();
2328 parent
2329 .named_children(&mut cursor)
2330 .filter(|child| child.kind() == "decorator")
2331 .filter_map(|decorator| decorator.named_child(0))
2332 .filter_map(expression_name_node)
2333 .any(|name| py_node_text(name, source).trim() == decorator_name)
2334}
2335
2336pub fn python_function_has_property_decorator(node: Node<'_>, source: &str) -> bool {
2344 python_function_has_decorator(node, source, "property")
2345 || python_function_has_decorator(node, source, "cached_property")
2346}
2347
2348pub fn python_first_parameter_name(node: Node<'_>, source: &str) -> Option<String> {
2351 let parameters = node.child_by_field_name("parameters")?;
2352 let mut cursor = parameters.walk();
2353 parameters
2354 .named_children(&mut cursor)
2355 .find_map(|child| python_parameter_name(child, source))
2356}
2357
2358pub fn collect_python_identifiers(node: Node<'_>, source: &str, identifiers: &mut HashSet<String>) {
2359 brokk_bifrost_core::analyzer::tree_walk::collect_trimmed_node_texts(
2360 node,
2361 source,
2362 identifiers,
2363 |node| node.kind() == "identifier",
2364 );
2365}
2366
2367pub fn parse_python_tree(source: &str) -> Option<Tree> {
2368 let mut parser = Parser::new();
2369 parser
2370 .set_language(&tree_sitter_python::LANGUAGE.into())
2371 .expect("failed to load python parser");
2372 parser.parse(source, None)
2373}
2374
2375#[cfg(test)]
2376mod primary_tests {
2377 use super::*;
2378 use brokk_bifrost_core::analyzer::python_facts::PythonAnnotationReferenceName;
2379
2380 fn parse(source: &str) -> ParsedFile {
2381 let file = ProjectFile::new(std::env::temp_dir(), "m4.py");
2382 let tree = parse_python_tree(source).expect("Python fixture tree");
2383 parse_python_file(
2384 &brokk_bifrost_core::analyzer::project::TestProject::new(
2385 file.root(),
2386 brokk_bifrost_core::analyzer::Language::Python,
2387 ),
2388 &file,
2389 source,
2390 &tree,
2391 )
2392 }
2393
2394 fn parse_native(source: &str) -> ParsedFile {
2395 let file = ProjectFile::new(std::env::temp_dir(), "m4.py");
2396 let tree = parse_python_tree(source).expect("Python fixture tree");
2397 parse_python_file_with_native_facts(
2398 &brokk_bifrost_core::analyzer::project::TestProject::new(
2399 file.root(),
2400 brokk_bifrost_core::analyzer::Language::Python,
2401 ),
2402 &file,
2403 source,
2404 &tree,
2405 )
2406 }
2407
2408 #[test]
2409 #[brokk_bifrost_test_macros::hardwon(
2410 "6b123df7cc68: quicksilver round 6 on brokk-bifrost-python: nested assignment targets recursively exhausted the stack"
2411 )]
2412 fn qs6_nested_unpacking_is_stack_safe() {
2413 let depth = 1024;
2414 let source = format!(
2415 "{}self.first, self.last{} = pair\n",
2416 "[".repeat(depth),
2417 "]".repeat(depth)
2418 );
2419 let tree = parse_python_tree(&source).unwrap();
2420 assert!(!tree.root_node().has_error());
2421 let assignment = tree
2422 .root_node()
2423 .named_child(0)
2424 .unwrap()
2425 .named_child(0)
2426 .unwrap();
2427 let target = assignment.child_by_field_name("left").unwrap();
2428 let attributes = std::thread::scope(|scope| {
2429 std::thread::Builder::new()
2430 .stack_size(64 * 1024)
2431 .spawn_scoped(scope, || {
2432 collect_self_assigned_attributes(target, &source, "self")
2433 })
2434 .unwrap()
2435 .join()
2436 .unwrap()
2437 });
2438 assert_eq!(
2439 attributes
2440 .iter()
2441 .map(|(name, _)| name.as_str())
2442 .collect::<Vec<_>>(),
2443 ["first", "last"]
2444 );
2445 }
2446
2447 #[test]
2448 fn qs6_setuptools_literal_dependency_boundary() {
2449 let source = "from setuptools import setup\nsetup(install_requires=('requests>=2',), setup_requires=['wheel'], tests_require=('pytest',), extras_require={'speed': ('orjson',)})\nsetup(install_requires=dynamic())\n";
2452 let arguments = setuptools_setup_py_literal_arguments(source).unwrap();
2453 let rendered = arguments
2454 .iter()
2455 .map(|argument| {
2456 format!(
2457 "{} {} {:?} {}",
2458 argument.call_index,
2459 argument.name,
2460 argument.value,
2461 &source[argument.byte_range.clone()]
2462 )
2463 })
2464 .collect::<Vec<_>>()
2465 .join("\n");
2466 assert_eq!(
2467 rendered,
2468 concat!(
2469 "0 install_requires Some(Strings([\"requests>=2\"])) ('requests>=2',)\n",
2470 "0 setup_requires Some(Strings([\"wheel\"])) ['wheel']\n",
2471 "0 tests_require Some(Strings([\"pytest\"])) ('pytest',)\n",
2472 "0 extras_require Some(Extras([(\"speed\", [\"orjson\"])])) {'speed': ('orjson',)}\n",
2473 "1 install_requires None dynamic()",
2474 )
2475 );
2476 }
2477
2478 #[test]
2479 #[brokk_bifrost_test_macros::hardwon(
2480 "#3673: declarations ignored an overlay-only setuptools manifest"
2481 )]
2482 fn snapshot_root_declaration_witness() {
2483 use brokk_bifrost_core::analyzer::Language;
2484 use brokk_bifrost_core::analyzer::project::{OverlayProject, Project, TestProject};
2485 use std::sync::Arc;
2486 let root = tempfile::tempdir().unwrap();
2487 let disk = Arc::new(TestProject::new(root.path(), Language::Python));
2488 let overlay = OverlayProject::new(disk);
2489 let manifest = ProjectFile::new(root.path(), "pyproject.toml");
2490 assert!(overlay.set(
2491 manifest.abs_path(),
2492 "[tool.setuptools.packages.find]\nwhere = ['source']\n".to_string()
2493 ));
2494 let file = ProjectFile::new(root.path(), "source/pkg/model.py");
2495 assert!(overlay.set(file.abs_path(), "class Widget:\n pass\n".to_string()));
2496 let snapshot = overlay.snapshot();
2497 assert!(snapshot.read_source(&manifest).unwrap().contains("source"));
2498 let source = snapshot.read_source(&file).unwrap();
2499 let tree = parse_python_tree(&source).unwrap();
2500 let parsed = parse_python_file(&snapshot, &file, &source, &tree);
2501 let names: Vec<_> = parsed
2502 .declarations()
2503 .iter()
2504 .map(CodeUnit::fq_name)
2505 .collect();
2506 assert!(names.contains(&"pkg.model.Widget".to_string()), "{names:?}");
2507 }
2508
2509 #[test]
2510 fn setup_py_unwrap_parenthesized_skips_comments() {
2511 let source = "VALUE = (\n # before\n \"value\" # after\n)\n";
2512 let tree = parse_python_tree(source).expect("Python fixture tree");
2513 let mut pending = vec![tree.root_node()];
2514 let parenthesized = loop {
2515 let node = pending.pop().expect("parenthesized expression exists");
2516 if node.kind() == "parenthesized_expression" {
2517 break node;
2518 }
2519 let mut cursor = node.walk();
2520 pending.extend(node.named_children(&mut cursor));
2521 };
2522
2523 assert_eq!(
2524 setup_py_unwrap_parenthesized(parenthesized).kind(),
2525 "string"
2526 );
2527 }
2528
2529 #[test]
2530 #[brokk_bifrost_test_macros::hardwon(
2531 "f4495e5c7c62: replacing a duplicate Python class left stale descendant callable metadata links"
2532 )]
2533 fn replacing_a_class_withdraws_descendant_metadata_links() {
2534 let parsed = parse_native(
2535 "from typing import overload\nclass Replaced:\n def obsolete(self): pass\n @overload\n def current(self): pass\n class Nested:\n def stale(self): pass\n\nclass Kept:\n def stable(self): pass\n\nclass Replaced:\n def current(self): pass\n",
2536 );
2537 for bridge in &parsed.resolution_facts.definition_units {
2538 assert!(
2539 parsed.contains_declaration(&bridge.unit),
2540 "native definitions must not resurrect withdrawn descendants: {bridge:?}"
2541 );
2542 }
2543 let mut names = parsed
2544 .source_declaration_metadata
2545 .iter()
2546 .map(|link| {
2547 assert!(
2548 parsed.signature_metadata[&link.unit]
2549 .get(link.metadata_ordinal)
2550 .is_some(),
2551 "the link must name retained callable metadata: {link:?}"
2552 );
2553 link.unit.short_name().to_string()
2554 })
2555 .collect::<Vec<_>>();
2556 names.sort();
2557 assert_eq!(names, ["Kept.stable", "Replaced.current"]);
2558 let current = parsed
2559 .source_declaration_metadata
2560 .iter()
2561 .find(|link| link.unit.short_name() == "Replaced.current")
2562 .unwrap();
2563 assert!(
2564 !parsed.signature_metadata[¤t.unit][current.metadata_ordinal]
2565 .is_declaration_only(),
2566 "a withdrawn overload must not mark the new implementation"
2567 );
2568 assert_eq!(
2569 parsed
2570 .source_declaration_units
2571 .iter()
2572 .filter(|(_, unit)| unit.short_name() == "Replaced.current")
2573 .count(),
2574 1,
2575 "the new method replaces even an already-withdrawn source link"
2576 );
2577 }
2578
2579 #[test]
2580 fn canonical_declarations_share_structural_name_identity_and_exact_ranges() {
2581 let source = "@decorate\nclass Item:\n value = 1\n value = 2\n def write(self):\n self.member = 1\n self.member = 2\n\nfirst, second = 1, 2\n";
2582 let parsed = parse(source);
2583 let facts = parsed
2584 .source_facts
2585 .as_ref()
2586 .expect("canonical Python facts");
2587 for (id, unit) in &parsed.source_declaration_units {
2588 let declaration = facts.occurrences.declaration(*id);
2589 let range = facts.occurrences.occurrence(declaration.occurrence).range;
2590 assert!(parsed.ranges[unit].contains(&range), "{unit:?}: {range:?}");
2591 let name = declaration.name.expect("source declaration name");
2592 assert!(
2593 facts
2594 .structural
2595 .nodes()
2596 .iter()
2597 .any(|node| node.occurrence == name),
2598 "shared name identity for {unit:?}"
2599 );
2600 }
2601 let fields = parsed
2602 .source_declaration_units
2603 .iter()
2604 .filter(|(_, unit)| unit.short_name() == "Item.value")
2605 .collect::<Vec<_>>();
2606 assert_eq!(fields.len(), 2);
2607 assert_ne!(fields[0].0, fields[1].0);
2608 assert_eq!(
2609 parsed
2610 .source_declaration_units
2611 .iter()
2612 .filter(|(_, unit)| unit.short_name() == "Item.member")
2613 .count(),
2614 1
2615 );
2616 let class = parsed
2617 .source_declaration_units
2618 .iter()
2619 .find(|(_, unit)| unit.short_name() == "Item")
2620 .expect("decorated class");
2621 let range = facts
2622 .occurrences
2623 .occurrence(facts.occurrences.declaration(class.0).occurrence)
2624 .range;
2625 assert_eq!(range.start_byte, 0);
2626 assert_eq!(&source[range.start_byte..range.start_byte + 9], "@decorate");
2627 }
2628
2629 #[test]
2630 fn imports_keep_scope_order_and_late_overload_capture_without_local_pollution() {
2631 let source = "@overload\ndef public(value: int):\n pass\nfrom typing import overload\n\ndef outer():\n from typing import overload as local_only\n@local_only\ndef ordinary(value: int):\n pass\nmatch 1:\n case 1:\n from .models import Item as Imported\n";
2632 let parsed = parse(source);
2633 let facts = parsed.source_facts.as_ref().expect("canonical imports");
2634 assert_eq!(facts.imports.len(), 3);
2635 assert!(
2636 facts.imports[0]
2637 .path
2638 .as_ref()
2639 .unwrap()
2640 .lexical_scopes
2641 .is_empty()
2642 );
2643 assert_eq!(
2644 facts.imports[1].path.as_ref().unwrap().lexical_scopes.len(),
2645 1
2646 );
2647 assert!(
2648 facts.imports[2]
2649 .path
2650 .as_ref()
2651 .unwrap()
2652 .lexical_scopes
2653 .is_empty()
2654 );
2655 for (id, import) in facts.generic_imports.iter().zip(&parsed.imports) {
2656 assert_eq!(
2657 &facts.imports[id.index()].import_info(&facts.occurrences),
2658 import
2659 );
2660 }
2661 let metadata = |name: &str| {
2662 parsed
2663 .signature_metadata
2664 .iter()
2665 .find(|(unit, _)| unit.short_name() == name)
2666 .unwrap()
2667 .1
2668 };
2669 assert!(metadata("public")[0].is_declaration_only());
2670 assert!(!metadata("ordinary")[0].is_declaration_only());
2671 }
2672
2673 #[test]
2674 #[brokk_bifrost_test_macros::hardwon(
2675 "1fa35d421feb: aliased import leaves pointed at imported-name ranges instead of the whole import statement"
2676 )]
2677 fn aliased_import_leaves_keep_the_statement_and_exact_binder_tokens() {
2678 let source = "from pkg import alpha as beta, beta as alpha\n";
2679 let parsed = parse(source);
2680 assert_eq!(parsed.imports.len(), 2);
2681 for (import, name) in parsed.imports.iter().zip(["beta", "alpha"]) {
2682 assert_eq!(import.path.as_ref().unwrap().declaration_start_byte, 0);
2683 let span = import.binder_span.as_ref().expect("exact alias binder");
2684 assert_eq!(&source[span.start_byte..span.end_byte], name);
2685 }
2686 }
2687
2688 #[test]
2689 fn parameter_offsets_use_ast_tokens_instead_of_annotation_spelling() {
2690 let source = "@decorate(\"(first, second)\")\ndef call(first: Literal[\"second\"], second: int):\n pass\n";
2691 let parsed = parse(source);
2692 let metadata = parsed
2693 .signature_metadata
2694 .values()
2695 .flatten()
2696 .find(|metadata| metadata.parameters().len() == 2)
2697 .expect("parameter metadata");
2698 let second = &metadata.parameters()[1];
2699 assert_eq!(
2700 &metadata.label()[second.start_byte()..second.end_byte()],
2701 "second"
2702 );
2703 assert_eq!(
2704 second.start_byte(),
2705 metadata.label().rfind("second:").unwrap()
2706 );
2707 }
2708
2709 #[test]
2710 fn multiline_parameter_metadata_keeps_each_formal_name_at_its_ast_offset() {
2711 let source = concat!(
2712 "@decorate\n",
2713 "def call(\n",
2714 " bare,\n",
2715 " typed: T,\n",
2716 " default=1,\n",
2717 " typed_default: U = 2,\n",
2718 " *args: V,\n",
2719 " required_kw,\n",
2720 " **kwargs: W,\n",
2721 "):\n",
2722 " pass\n",
2723 );
2724 let parsed = parse(source);
2725 let metadata = parsed
2726 .signature_metadata
2727 .values()
2728 .flatten()
2729 .find(|metadata| metadata.parameters().len() == 7)
2730 .expect("multiline callable parameter metadata");
2731 let names = metadata
2732 .parameters()
2733 .iter()
2734 .map(|parameter| &metadata.label()[parameter.start_byte()..parameter.end_byte()])
2735 .collect::<Vec<_>>();
2736 assert_eq!(
2737 names,
2738 [
2739 "bare",
2740 "typed",
2741 "default",
2742 "typed_default",
2743 "args",
2744 "required_kw",
2745 "kwargs"
2746 ]
2747 );
2748 assert!(metadata.label().contains(" typed_default: U = 2,"));
2749 assert!(metadata.label().ends_with(" ..."));
2750 }
2751
2752 #[test]
2753 fn crlf_multiline_parameter_metadata_keeps_each_formal_name_at_its_ast_offset() {
2754 let source = concat!(
2755 "@decorate\r\n",
2756 "def call(\r\n",
2757 " bare,\r\n",
2758 " typed: T,\r\n",
2759 " default=1,\r\n",
2760 "):\r\n",
2761 " pass\r\n",
2762 );
2763 let parsed = parse(source);
2764 let metadata = parsed
2765 .signature_metadata
2766 .values()
2767 .flatten()
2768 .find(|metadata| metadata.parameters().len() == 3)
2769 .expect("multiline callable parameter metadata");
2770 let names = metadata
2771 .parameters()
2772 .iter()
2773 .map(|parameter| &metadata.label()[parameter.start_byte()..parameter.end_byte()])
2774 .collect::<Vec<_>>();
2775 assert_eq!(names, ["bare", "typed", "default"]);
2776 }
2777
2778 #[test]
2779 fn crlf_parameter_metadata_preserves_normalized_signature_offsets() {
2780 for indentation in ["", " "] {
2781 let header = concat!(
2782 "@decorate\r\n",
2783 "def call(\r\n",
2784 " first: T,\r\n",
2785 " second: U = 2,\r\n",
2786 "):\r\n",
2787 " pass\r\n",
2788 );
2789 let source = if indentation.is_empty() {
2790 header.to_owned()
2791 } else {
2792 format!(
2793 "class Service:\r\n{}",
2794 header
2795 .split_inclusive('\n')
2796 .map(|line| format!("{indentation}{line}"))
2797 .collect::<String>()
2798 )
2799 };
2800 let parsed = parse(&source);
2801 let metadata = parsed
2802 .signature_metadata
2803 .values()
2804 .flatten()
2805 .find(|metadata| metadata.parameters().len() == 2)
2806 .expect("CRLF callable metadata");
2807 assert!(!metadata.label().contains('\r'));
2808 let labels = metadata
2809 .parameters()
2810 .iter()
2811 .map(|parameter| &metadata.label()[parameter.start_byte()..parameter.end_byte()])
2812 .collect::<Vec<_>>();
2813 assert_eq!(labels, ["first", "second"]);
2814 }
2815 }
2816
2817 #[test]
2818 #[brokk_bifrost_test_macros::hardwon(
2819 "#4455: multiline Python parameter offsets dropped a newline before an unindented label"
2820 )]
2821 fn multiline_parameter_offsets_follow_ast_ranges_through_header_normalization() {
2822 let source = concat!(
2823 "def manyargs(first: Annotated[int, 'second'] = second, second: int = 1,\r\n",
2824 "# comment between parameters\r\n",
2825 "third: int = second,\r\n",
2826 "/,\r\n",
2827 "*args: object,\r\n",
2828 "keyword: int = 2,\r\n",
2829 "**kwargs: object,\r\n",
2830 "):\r\n",
2831 " pass\r\n",
2832 "def keyword_only(\r\n",
2833 "positional,\r\n",
2834 "*,\r\n",
2835 "required,\r\n",
2836 "):\r\n",
2837 " pass\r\n",
2838 "class Service:\n",
2839 "\tdef tabbed(\n",
2840 "\t\tfirst,\n",
2841 "\t\tsecond,\n",
2842 "\t):\n",
2843 "\t\tpass\n",
2844 );
2845 let parsed = parse(source);
2846 let metadata = |name: &str| {
2847 parsed
2848 .signature_metadata
2849 .iter()
2850 .find(|(unit, _)| unit.short_name() == name)
2851 .unwrap_or_else(|| {
2852 let available = parsed
2853 .signature_metadata
2854 .keys()
2855 .map(|unit| unit.short_name())
2856 .collect::<Vec<_>>();
2857 panic!("callable metadata for {name}; available names: {available:?}");
2858 })
2859 .1
2860 .first()
2861 .expect("one signature")
2862 };
2863 let parameter_names = |name: &str| {
2864 let metadata = metadata(name);
2865 metadata
2866 .parameters()
2867 .iter()
2868 .map(|parameter| &metadata.label()[parameter.start_byte()..parameter.end_byte()])
2869 .collect::<Vec<_>>()
2870 };
2871
2872 let manyargs = metadata("manyargs");
2873 assert_eq!(
2874 parameter_names("manyargs"),
2875 ["first", "second", "third", "args", "keyword", "kwargs"]
2876 );
2877 assert!(
2878 manyargs
2879 .label()
2880 .contains("# comment between parameters\nthird: int")
2881 );
2882 assert!(manyargs.label().contains("\n/,\n*args: object"));
2883 assert!(!manyargs.label().contains('\r'));
2884 assert_eq!(parameter_names("keyword_only"), ["positional", "required"]);
2885 assert_eq!(parameter_names("Service.tabbed"), ["first", "second"]);
2886 }
2887
2888 #[test]
2889 fn empty_and_recovered_python_publish_canonical_structural_facts() {
2890 for source in [
2891 "",
2892 "class Broken(:\n pass\ndef valid(value):\n return value\n",
2893 "def broken(:\n pass\n",
2894 ] {
2895 let parsed = parse(source);
2896 let facts = parsed.source_facts.expect("complete producer publication");
2897 assert_eq!(facts.source_bytes, source.len());
2898 assert!(facts.python.is_some());
2899 assert_eq!(parsed.resolution_facts, Default::default());
2900 assert!(facts.native_site_occurrences.is_empty());
2901 assert!(facts.native_declaration_sources.is_empty());
2902 assert!(
2903 facts
2904 .occurrences
2905 .occurrences()
2906 .iter()
2907 .all(|occurrence| occurrence.range.end_byte <= source.len())
2908 );
2909 }
2910 }
2911
2912 #[test]
2913 fn unmounted_annotated_functions_keep_source_declarations_and_return_facts() {
2914 let source = "if outer:\n if inner:\n def hidden(value: int) -> Hidden:\n pass\n\ndef visible() -> Visible:\n pass\n";
2915 let parsed = parse(source);
2916 let facts = parsed
2917 .source_facts
2918 .as_ref()
2919 .expect("canonical Python facts");
2920 let python = facts.python.as_ref().expect("Python source facts");
2921 assert_eq!(python.callable_returns.len(), 2);
2922
2923 let fact_named = |name: &str| {
2924 python
2925 .callable_returns
2926 .iter()
2927 .find(|fact| {
2928 let declaration = facts.occurrences.declaration(fact.declaration);
2929 let Some(name_occurrence) = declaration.name else {
2930 return false;
2931 };
2932 let range = facts.occurrences.occurrence(name_occurrence).range;
2933 &source[range.start_byte..range.end_byte] == name
2934 })
2935 .expect("callable return fact")
2936 };
2937 let hidden = fact_named("hidden");
2938 let hidden_declaration = facts.occurrences.declaration(hidden.declaration);
2939 let hidden_range = facts
2940 .occurrences
2941 .occurrence(hidden_declaration.occurrence)
2942 .range;
2943 assert_eq!(
2944 &source[hidden_range.start_byte..hidden_range.start_byte + 3],
2945 "def"
2946 );
2947 let hidden_annotation = hidden.return_annotation.expect("hidden annotation");
2948 let hidden_annotation_range = facts.occurrences.occurrence(hidden_annotation).range;
2949 assert_eq!(
2950 &source[hidden_annotation_range.start_byte..hidden_annotation_range.end_byte],
2951 "Hidden"
2952 );
2953 assert!(
2954 parsed
2955 .source_declaration_units
2956 .iter()
2957 .all(|(_, unit)| unit.short_name() != "hidden")
2958 );
2959
2960 let visible = fact_named("visible");
2961 assert!(visible.return_annotation.is_some());
2962 assert!(visible.runtime_type.is_some());
2963 assert!(
2964 parsed
2965 .source_declaration_units
2966 .iter()
2967 .any(|(_, unit)| unit.short_name() == "visible")
2968 );
2969 let visible_metadata = parsed
2970 .signature_metadata
2971 .iter()
2972 .find(|(unit, _)| unit.short_name() == "visible")
2973 .and_then(|(_, metadata)| metadata.first())
2974 .expect("visible signature metadata");
2975 assert_eq!(visible_metadata.return_type_text(), Some("Visible"));
2976 assert_eq!(
2977 visible_metadata.return_type_identity(),
2978 visible.runtime_type.as_ref()
2979 );
2980 }
2981
2982 #[test]
2983 fn return_fact_preserves_unsupported_annotation_without_runtime_identity() {
2984 let source = "def unsupported() -> Left | Right:\n pass\n\ndef plain():\n pass\n";
2985 let parsed = parse(source);
2986 let facts = parsed
2987 .source_facts
2988 .as_ref()
2989 .expect("canonical Python facts");
2990 let python = facts.python.as_ref().expect("Python source facts");
2991 let by_name = |name: &str| {
2992 python
2993 .callable_returns
2994 .iter()
2995 .find(|fact| {
2996 let declaration = facts.occurrences.declaration(fact.declaration);
2997 let Some(name_occurrence) = declaration.name else {
2998 return false;
2999 };
3000 let range = facts.occurrences.occurrence(name_occurrence).range;
3001 &source[range.start_byte..range.end_byte] == name
3002 })
3003 .expect("callable return fact")
3004 };
3005 let unsupported = by_name("unsupported");
3006 assert!(unsupported.return_annotation.is_some());
3007 assert!(unsupported.runtime_type.is_none());
3008 let plain = by_name("plain");
3009 assert!(plain.return_annotation.is_none());
3010 assert!(plain.runtime_type.is_none());
3011 }
3012
3013 #[test]
3014 #[brokk_bifrost_test_macros::hardwon(
3015 "45e71db39f13: generic Python annotations mistook a list element for the returned class owner"
3016 )]
3017 fn return_annotation_references_preserve_runtime_owners() {
3018 let source = "def generic() -> list[User]:\n pass\n\ndef nested() -> list[pkg.User]:\n pass\n\ndef direct() -> pkg.User:\n pass\n\ndef quoted() -> \"pkg.User\":\n pass\n\ndef unsupported() -> Left | Right:\n pass\n";
3019 let parsed = parse(source);
3020 let facts = parsed
3021 .source_facts
3022 .as_ref()
3023 .expect("canonical Python facts");
3024 let python = facts.python.as_ref().expect("Python source facts");
3025 let fact_named = |name: &str| {
3026 python
3027 .callable_returns
3028 .iter()
3029 .find(|fact| {
3030 let declaration = facts.occurrences.declaration(fact.declaration);
3031 let Some(name_occurrence) = declaration.name else {
3032 return false;
3033 };
3034 let range = facts.occurrences.occurrence(name_occurrence).range;
3035 &source[range.start_byte..range.end_byte] == name
3036 })
3037 .expect("callable return fact")
3038 };
3039 let generic = fact_named("generic");
3040 assert_eq!(
3041 generic
3042 .annotation_references
3043 .iter()
3044 .map(|reference| &reference.name)
3045 .collect::<Vec<_>>(),
3046 vec![&PythonAnnotationReferenceName::Lexical("list".to_string())]
3047 );
3048
3049 let nested = fact_named("nested");
3051 assert_eq!(nested.annotation_references.len(), 1);
3052 assert!(matches!(
3053 &nested.annotation_references[0].name,
3054 PythonAnnotationReferenceName::Lexical(name) if name == "list"
3055 ));
3056
3057 let direct = fact_named("direct");
3058 assert_eq!(direct.annotation_references.len(), 3);
3059 assert_eq!(direct.annotation_references[0].subtree_end, 3);
3060 assert_eq!(direct.annotation_references[0].lookup_depth, 1);
3061 assert!(matches!(
3062 &direct.annotation_references[0].name,
3063 PythonAnnotationReferenceName::Qualified(path)
3064 if path == &vec!["pkg".to_string(), "User".to_string()]
3065 ));
3066
3067 let quoted = fact_named("quoted");
3068 assert_eq!(quoted.annotation_references.len(), 1);
3069 assert!(matches!(
3070 "ed.annotation_references[0].name,
3071 PythonAnnotationReferenceName::Lexical(name) if name == "pkg.User"
3072 ));
3073 assert_eq!(quoted.annotation_references[0].lookup_depth, 2);
3074
3075 let unsupported = fact_named("unsupported");
3076 assert!(
3077 unsupported
3078 .annotation_references
3079 .iter()
3080 .all(|reference| !matches!(
3081 &reference.name,
3082 PythonAnnotationReferenceName::Qualified(_)
3083 ))
3084 );
3085 }
3086}
3087
3088fn python_chained_assignment_targets<'tree>(assignment: Node<'tree>) -> Vec<Node<'tree>> {
3095 super::imports::python_assignment_chain(assignment).0
3096}
3097
3098#[cfg(test)]
3099mod supertype_tests {
3100 use super::extract_python_supertypes;
3101 use tree_sitter::Node;
3102
3103 fn class_node<'tree>(tree: &'tree tree_sitter::Tree, source: &str) -> Node<'tree> {
3104 let mut cursor = tree.root_node().walk();
3105 tree.root_node()
3106 .named_children(&mut cursor)
3107 .find(|node| node.kind() == "class_definition")
3108 .unwrap_or_else(|| panic!("source declares a class: {source}"))
3109 }
3110
3111 fn parse(source: &str) -> tree_sitter::Tree {
3112 super::parse_python_tree(source).expect("the source parses")
3113 }
3114
3115 #[test]
3116 #[brokk_bifrost_test_macros::hardwon(
3117 "50c1574b2070: unsupported Python positional bases were dropped and created false complete absence"
3118 )]
3119 fn every_positional_base_contributes_a_spelling() {
3120 for (source, expected) in [
3121 ("class A(Base): pass\n", vec!["Base"]),
3122 ("class A(pkg.Base): pass\n", vec!["pkg.Base"]),
3123 ("class A(Base[int]): pass\n", vec!["Base"]),
3126 ("class A(pkg.Base[str, int]): pass\n", vec!["pkg.Base"]),
3127 (
3128 "class A(Mapping[str, Any], Base): pass\n",
3129 vec!["Mapping", "Base"],
3130 ),
3131 ("class A(Base, metaclass=Meta): pass\n", vec!["Base"]),
3133 ("class A(metaclass=Meta): pass\n", Vec::new()),
3134 (
3137 "class A(namedtuple(\"P\", \"x\")): pass\n",
3138 vec!["namedtuple(\"P\", \"x\")"],
3139 ),
3140 ("class A(*bases): pass\n", vec!["*bases"]),
3141 ("class A: pass\n", Vec::new()),
3142 ] {
3143 let tree = parse(source);
3144 let node = class_node(&tree, source);
3145 assert_eq!(
3146 extract_python_supertypes(node, source),
3147 expected,
3148 "supertypes of {source}"
3149 );
3150 }
3151 }
3152}
3153
3154#[cfg(test)]
3155mod callable_modifier_tests {
3156 use super::*;
3157
3158 #[test]
3181 #[brokk_bifrost_test_macros::hardwon(
3182 "#3451: Python callable receiver modifiers were missing, leaving workspace procedures unkeyable"
3183 )]
3184 fn callable_metadata_records_python_receiver_contracts_structurally() {
3185 let source = "def free(value):\n return value\n\nclass Widget:\n def __init__(self, spec):\n self.spec = spec\n\n def render(self, target):\n return target\n\n @staticmethod\n def build(spec):\n return spec\n\n @classmethod\n def measure(cls, target):\n return target\n\n @property\n def label(self):\n return \"widget\"\n\nasync def fetch_all(url):\n return url\n\ndef outer(seed):\n def inner(value):\n return value\n return inner(seed)\n\nassigned = lambda value: value\n";
3186 let file = ProjectFile::new(std::env::temp_dir(), "widget.py");
3187 let tree = parse_python_tree(source).expect("parse the Python fixture");
3188 let parsed = parse_python_file(
3189 &brokk_bifrost_core::analyzer::project::TestProject::new(
3190 file.root(),
3191 brokk_bifrost_core::analyzer::Language::Python,
3192 ),
3193 &file,
3194 source,
3195 &tree,
3196 );
3197
3198 let entry = |fq_name: &str| {
3199 parsed
3200 .signature_metadata
3201 .iter()
3202 .find(|(unit, _)| unit.fq_name() == fq_name)
3203 .unwrap_or_else(|| {
3204 panic!(
3205 "missing Python declaration {fq_name}; recorded {:?}",
3206 parsed
3207 .signature_metadata
3208 .keys()
3209 .map(CodeUnit::fq_name)
3210 .collect::<Vec<_>>()
3211 )
3212 })
3213 };
3214 let modifiers = |fq_name: &str| {
3215 let (_, entries) = entry(fq_name);
3216 let metadata = entries
3217 .first()
3218 .unwrap_or_else(|| panic!("{fq_name} carries no signature metadata"));
3219 assert!(
3220 metadata.callable_modifiers_recorded(),
3221 "{fq_name} must record that the walk read its declaration shape"
3222 );
3223 (
3224 metadata.callable_is_static(),
3225 metadata.callable_is_constructor(),
3226 metadata.parameters().len(),
3227 )
3228 };
3229
3230 assert_eq!(modifiers("widget.free"), (false, false, 1));
3231 assert_eq!(
3232 modifiers("widget.Widget.__init__"),
3233 (false, false, 2),
3234 "`__init__` is an ordinary instance method, exactly as Ruby's `initialize`"
3235 );
3236 assert_eq!(modifiers("widget.Widget.render"), (false, false, 2));
3237 assert_eq!(
3238 modifiers("widget.Widget.build"),
3239 (true, false, 1),
3240 "`@staticmethod` binds no instance receiver"
3241 );
3242 assert_eq!(
3243 modifiers("widget.Widget.measure"),
3244 (true, false, 2),
3245 "`@classmethod` receives the class, not an instance"
3246 );
3247 assert_eq!(
3248 modifiers("widget.fetch_all"),
3249 (false, false, 1),
3250 "an async def is the same function_definition node"
3251 );
3252 assert_eq!(modifiers("widget.outer"), (false, false, 1));
3253
3254 let (property_unit, property_entries) = entry("widget.Widget.label");
3255 assert_eq!(
3256 property_unit.kind(),
3257 CodeUnitType::Field,
3258 "`@property` is read as an attribute, so it mints a Field and no key"
3259 );
3260 assert!(!property_unit.is_callable());
3261 assert!(
3262 property_entries
3263 .first()
3264 .expect("the property still carries signature metadata")
3265 .callable_modifiers_recorded(),
3266 "the shared metadata path records modifiers for every declaration it mints"
3267 );
3268
3269 let lambda_unit = parsed
3274 .declarations()
3275 .iter()
3276 .find(|unit| unit.fq_name() == "widget.assigned")
3277 .expect("the module walk still records the name-bound lambda's assignment");
3278 assert_eq!(
3279 lambda_unit.kind(),
3280 CodeUnitType::Field,
3281 "a name-bound lambda is recorded as a field, not as a callable declaration"
3282 );
3283 assert!(
3284 !parsed.signature_metadata.contains_key(lambda_unit),
3285 "a field assignment carries signature text but no callable signature metadata"
3286 );
3287 assert!(
3288 parsed
3289 .declarations()
3290 .iter()
3291 .all(|unit| unit.terminal_name() != "inner"),
3292 "a function nested in another function is a local binding, not a declaration"
3293 );
3294 }
3295}