use anyhow::Result;
use regex::Regex;
use std::collections::HashSet;
use std::fs;
use std::path::{Path, PathBuf};
use std::sync::OnceLock;
use crate::config::PythonAnalyzerConfig;
use crate::errors;
use crate::file_index::FileIndex;
use crate::graph::Product;
use super::{DepAnalyzer, ScanResult};
struct Import {
module: String,
names: Vec<String>,
}
fn import_items(list: &str) -> impl Iterator<Item = String> + '_ {
list.split(',')
.filter_map(|item| item.split_whitespace().next())
.filter(|name| is_dotted_name(name))
.map(str::to_string)
}
fn is_dotted_name(name: &str) -> bool {
!name.is_empty()
&& name.split('.').all(|part| {
let mut chars = part.chars();
chars.next().is_some_and(|c| c.is_alphabetic() || c == '_')
&& chars.all(|c| c.is_alphanumeric() || c == '_')
})
}
fn outside_triple_quotes(line: &str, open: &mut Option<&'static str>) -> String {
const QUOTES: [&str; 2] = ["\"\"\"", "'''"];
let mut out = String::new();
let mut rest = line;
loop {
if let Some(quote) = *open {
let Some(end) = rest.find(quote) else {
return out;
};
rest = &rest[end + quote.len()..];
*open = None;
} else {
let next = QUOTES
.iter()
.filter_map(|quote| rest.find(quote).map(|at| (at, *quote)))
.min_by_key(|&(at, _)| at);
let Some((at, quote)) = next else {
out.push_str(rest);
return out;
};
out.push_str(&rest[..at]);
rest = &rest[at + quote.len()..];
*open = Some(quote);
}
}
}
fn parse_imports(content: &str) -> Vec<Import> {
static FROM_RE: OnceLock<Regex> = OnceLock::new();
static IMPORT_RE: OnceLock<Regex> = OnceLock::new();
let from_re = FROM_RE.get_or_init(|| {
Regex::new(r"^\s*from\s+(\S+)\s+import\s+(.*)$").expect(errors::INVALID_REGEX)
});
let import_re =
IMPORT_RE.get_or_init(|| Regex::new(r"^\s*import\s+(.*)$").expect(errors::INVALID_REGEX));
let mut imports: Vec<Import> = Vec::new();
let mut in_parens = false;
let mut in_string: Option<&'static str> = None;
let lines: Vec<String> = content
.lines()
.map(|line| outside_triple_quotes(line, &mut in_string))
.collect();
let statements = lines
.iter()
.flat_map(|line| line.split('#').next().unwrap_or("").split(';'));
for code in statements {
if in_parens {
let (names, closed) = code
.split_once(')')
.map_or((code, false), |(n, _)| (n, true));
if let Some(last) = imports.last_mut() {
last.names.extend(import_items(names));
}
in_parens = !closed;
} else if let Some(caps) = from_re.captures(code) {
let rest = caps[2].trim();
let names = match rest.strip_prefix('(') {
Some(inner) => {
if let Some((names, _)) = inner.split_once(')') {
names
} else {
in_parens = true;
inner
}
}
None => rest,
};
imports.push(Import {
module: caps[1].to_string(),
names: import_items(names).collect(),
});
} else if let Some(caps) = import_re.captures(code) {
imports.extend(import_items(&caps[1]).map(|module| Import {
module,
names: Vec::new(),
}));
}
}
imports
}
pub fn scan_python_imports(source: &Path) -> Result<Vec<String>> {
let content = crate::errors::ctx(
fs::read_to_string(source),
&format!("Failed to read Python source: {}", source.display()),
)?;
Ok(parse_imports(&content)
.into_iter()
.map(|import| import.module)
.collect())
}
pub struct PythonDepAnalyzer {
config: PythonAnalyzerConfig,
}
impl PythonDepAnalyzer {
pub const fn new(config: PythonAnalyzerConfig) -> Self {
Self { config }
}
fn scan_imports(&self, source: &Path, file_index: &FileIndex) -> Result<ScanResult> {
let mut imports = Vec::new();
let mut absent = Vec::new();
let mut seen: HashSet<PathBuf> = HashSet::from([source.to_path_buf()]);
let mut queue: Vec<PathBuf> = vec![source.to_path_buf()];
while let Some(file) = queue.pop() {
let content = errors::ctx(
fs::read_to_string(&file),
&format!("Failed to read Python source: {}", file.display()),
)?;
for import in parse_imports(&content) {
let submodules = import.names.iter().map(|n| submodule(&import.module, n));
for module in std::iter::once(import.module.clone()).chain(submodules) {
let (found, probed) = self.resolve_module(&file, &module, file_index);
absent.extend(probed);
for path in found {
if seen.insert(path.clone()) {
imports.push(path.clone());
if path.is_file() {
queue.push(path);
}
}
}
}
}
}
absent.sort();
absent.dedup();
Ok(ScanResult::deps(imports, absent))
}
fn resolve_module(
&self,
importer: &Path,
module: &str,
file_index: &FileIndex,
) -> (Vec<PathBuf>, Vec<PathBuf>) {
let dots = module.len() - module.trim_start_matches('.').len();
let name = &module[dots..];
let parts: Vec<&str> = if name.is_empty() {
Vec::new()
} else {
name.split('.').collect()
};
if (dots == 0 && parts.is_empty()) || parts.iter().any(|p| p.is_empty()) {
return (Vec::new(), Vec::new());
}
let exists = |p: &Path| file_index.contains(p) || p.is_file();
if dots > 0 {
return match relative_base(importer, dots) {
Some(base) => Self::resolve_under(&base, &parts, &exists),
None => (Vec::new(), Vec::new()),
};
}
let roots = std::iter::once(crate::processor::parent_dir(importer).to_path_buf())
.chain(std::iter::once(PathBuf::new()))
.chain(self.config.search_paths.iter().map(PathBuf::from));
let mut probed = Vec::new();
for root in roots {
let (found, missing) = Self::resolve_under(&root, &parts, &exists);
probed.extend(missing);
if !found.is_empty() {
return (found, probed);
}
}
(Vec::new(), probed)
}
fn resolve_under(
root: &Path,
parts: &[&str],
exists: &impl Fn(&Path) -> bool,
) -> (Vec<PathBuf>, Vec<PathBuf>) {
let candidates = if parts.is_empty() {
vec![root.join("__init__.py")]
} else {
let module_path = parts.join("/");
vec![
root.join(format!("{module_path}.py")),
root.join(&module_path).join("__init__.py"),
]
};
let mut probed = Vec::new();
for candidate in candidates {
if !exists(&candidate) {
probed.push(candidate);
continue;
}
let mut found = vec![candidate];
for depth in 1..parts.len() {
let init = root.join(parts[..depth].join("/")).join("__init__.py");
if exists(&init) {
found.push(init);
} else {
probed.push(init);
}
}
return (found, probed);
}
(Vec::new(), probed)
}
}
fn submodule(module: &str, name: &str) -> String {
if module.ends_with('.') {
format!("{module}{name}")
} else {
format!("{module}.{name}")
}
}
fn relative_base(importer: &Path, dots: usize) -> Option<PathBuf> {
let mut base = importer.parent()?;
for _ in 1..dots {
base = base.parent()?;
}
Some(base.to_path_buf())
}
impl DepAnalyzer for PythonDepAnalyzer {
fn description(&self) -> &'static str {
"Scan Python source files for import dependencies"
}
fn enabled(&self) -> bool {
self.config.enabled
}
fn auto_detect(&self, file_index: &FileIndex) -> bool {
file_index.has_extension(".py")
}
fn match_product(&self, p: &Product) -> Option<PathBuf> {
if p.inputs.is_empty() {
return None;
}
let source = &p.inputs[0];
let ext = source.extension().and_then(|s| s.to_str()).unwrap_or("");
if ext == "py" {
Some(source.clone())
} else {
None
}
}
fn scan(
&self,
_ctx: &crate::build_context::BuildContext,
source: &Path,
file_index: &FileIndex,
) -> Result<ScanResult> {
self.scan_imports(source, file_index)
}
fn fingerprint_parts(&self, _ctx: &crate::build_context::BuildContext) -> Result<Vec<String>> {
super::config_fingerprint(&self.config)
}
}
inventory::submit! {
crate::registries::AnalyzerPlugin {
name: "python",
description: "Scan Python files for local import dependencies",
is_builtin: true,
version: 2,
create: |toml_value, _| {
let cfg: PythonAnalyzerConfig = crate::registries::deserialize_config(toml_value)?;
Ok(Box::new(PythonDepAnalyzer::new(cfg)))
},
defconfig_toml: || {
toml::to_string_pretty(&PythonAnalyzerConfig::default()).ok()
},
known_fields: crate::registries::typed_known_fields::<PythonAnalyzerConfig>,
}
}
#[cfg(test)]
mod tests {
use super::*;
fn parsed(src: &str) -> Vec<(String, Vec<String>)> {
parse_imports(src)
.into_iter()
.map(|i| (i.module, i.names))
.collect()
}
fn owned(names: &[&str]) -> Vec<String> {
names.iter().map(|s| (*s).to_string()).collect()
}
#[test]
fn parse_imports_statement_forms() {
let src = "\
import os, sys # trailing comment
import a.b as c
# import commented
import indented
from pkg.sub import x, y as z
from pkg import *
from multi import (
one,
two, # comment
)
from inline import (p, q)
from . import sibling
from .bar import baz
from ..pkg import qux
import first; import second
from third import t; x = 1
import fourth; fourth.path.insert(0, \"lib\")
doc = \"\"\"
import in_a_docstring
\"\"\"
one_line = '''import quoted'''; import fifth
";
assert_eq!(
parsed(src),
vec![
("os".to_string(), vec![]),
("sys".to_string(), vec![]),
("a.b".to_string(), vec![]),
("indented".to_string(), vec![]),
("pkg.sub".to_string(), owned(&["x", "y"])),
("pkg".to_string(), vec![]),
("multi".to_string(), owned(&["one", "two"])),
("inline".to_string(), owned(&["p", "q"])),
(".".to_string(), owned(&["sibling"])),
(".bar".to_string(), owned(&["baz"])),
("..pkg".to_string(), owned(&["qux"])),
("first".to_string(), vec![]),
("second".to_string(), vec![]),
("third".to_string(), owned(&["t"])),
("fourth".to_string(), vec![]),
("fifth".to_string(), vec![]),
]
);
}
#[test]
fn submodule_joins_after_a_bare_dot_without_a_second_dot() {
assert_eq!(submodule("pkg", "a"), "pkg.a");
assert_eq!(submodule(".bar", "baz"), ".bar.baz");
assert_eq!(submodule(".", "a"), ".a");
assert_eq!(submodule("..", "a"), "..a");
}
#[test]
fn relative_base_climbs_one_level_per_extra_dot() {
let importer = Path::new("pkg/sub/mod.py");
assert_eq!(relative_base(importer, 1), Some(PathBuf::from("pkg/sub")));
assert_eq!(relative_base(importer, 2), Some(PathBuf::from("pkg")));
assert_eq!(relative_base(importer, 3), Some(PathBuf::from("")));
assert_eq!(relative_base(importer, 4), None);
}
fn tree() -> FileIndex {
FileIndex::from_paths(
[
"top.py",
"pkg/__init__.py",
"pkg/bar.py",
"pkg/sub/__init__.py",
"pkg/sub/mod.py",
"pkg/sub/sibling.py",
"pkg/sub/deep/__init__.py",
"pkg/sub/deep/leaf.py",
]
.iter()
.map(PathBuf::from)
.collect(),
)
}
fn paths(names: &[&str]) -> Vec<PathBuf> {
names.iter().map(PathBuf::from).collect()
}
#[test]
fn python_resolves_single_dot_relative_imports() {
let analyzer = PythonDepAnalyzer::new(PythonAnalyzerConfig::default());
let index = tree();
let importer = Path::new("pkg/sub/mod.py");
let (found, probed) = analyzer.resolve_module(importer, ".", &index);
assert_eq!(found, paths(&["pkg/sub/__init__.py"]));
assert_eq!(probed, paths(&[]));
let (found, _) = analyzer.resolve_module(importer, ".sibling", &index);
assert_eq!(found, paths(&["pkg/sub/sibling.py"]));
let (found, _) = analyzer.resolve_module(importer, ".deep.leaf", &index);
assert_eq!(
found,
paths(&["pkg/sub/deep/leaf.py", "pkg/sub/deep/__init__.py"])
);
let (found, probed) = analyzer.resolve_module(importer, ".bar", &index);
assert_eq!(found, paths(&[]));
assert_eq!(
probed,
paths(&["pkg/sub/bar.py", "pkg/sub/bar/__init__.py"])
);
}
#[test]
fn python_resolves_multi_dot_relative_imports() {
let analyzer = PythonDepAnalyzer::new(PythonAnalyzerConfig::default());
let index = tree();
let importer = Path::new("pkg/sub/mod.py");
let (found, _) = analyzer.resolve_module(importer, "..bar", &index);
assert_eq!(found, paths(&["pkg/bar.py"]));
let (found, _) = analyzer.resolve_module(importer, "..", &index);
assert_eq!(found, paths(&["pkg/__init__.py"]));
let (found, _) = analyzer.resolve_module(importer, "...top", &index);
assert_eq!(found, paths(&["top.py"]));
let (found, probed) = analyzer.resolve_module(importer, "....top", &index);
assert_eq!(found, paths(&[]));
assert_eq!(probed, paths(&[]));
}
#[test]
fn python_resolves_absolute_imports_through_the_roots() {
let analyzer = PythonDepAnalyzer::new(PythonAnalyzerConfig::default());
let index = tree();
let importer = Path::new("pkg/sub/mod.py");
let (found, probed) = analyzer.resolve_module(importer, "top", &index);
assert_eq!(found, paths(&["top.py"]));
assert_eq!(
probed,
paths(&["pkg/sub/top.py", "pkg/sub/top/__init__.py"])
);
let (found, _) = analyzer.resolve_module(importer, "pkg.bar", &index);
assert_eq!(found, paths(&["pkg/bar.py", "pkg/__init__.py"]));
}
}