supercov_engine/
python_project.rs

1//! Python project discovery and ahead-of-run obligation preparation.
2//!
3//! The project runs in place: nothing here copies or rewrites sources. Rust
4//! reads every in-scope `.py` file once, builds the complete manifest and the
5//! runtime probe plan, and records which files were included, excluded or
6//! unparseable so the run can say exactly what its denominator covers.
7
8use std::{
9    collections::{BTreeMap, BTreeSet},
10    fs,
11    path::{Path, PathBuf},
12};
13
14use serde_json::json;
15
16use crate::{
17    coverage_report::CoverageManifest,
18    integrity::ExplicitIntegrityInputs,
19    python_instrumenter::{
20        PYTHON_PROBE_PLAN_VERSION, PythonFilePlan, PythonProbePlan, build_python_obligations,
21    },
22    source_discovery::{SourceScope, SourceScopeEntry, SourceScopeMode, SourceScopeStatus},
23};
24
25pub const UNPARSEABLE_LIMITATION: &str = "python-source-unparseable";
26
27/// Directories that never hold the project's own measured source.
28const EXCLUDED_DIRECTORIES: &[&str] = &[
29    ".git",
30    ".hg",
31    ".svn",
32    ".supercov",
33    ".mcdc-pool",
34    ".cache",
35    "node_modules",
36    "__pycache__",
37    ".venv",
38    "venv",
39    ".env",
40    "env",
41    ".tox",
42    ".nox",
43    ".mypy_cache",
44    ".pytest_cache",
45    ".ruff_cache",
46    ".hypothesis",
47    ".eggs",
48    "build",
49    "dist",
50    "target",
51    "site-packages",
52    "htmlcov",
53];
54
55const DEPENDENCY_FILES: &[&str] = &[
56    "pyproject.toml",
57    "setup.cfg",
58    "setup.py",
59    "requirements.txt",
60    "requirements-dev.txt",
61    "Pipfile",
62    "Pipfile.lock",
63    "poetry.lock",
64    "uv.lock",
65    "pdm.lock",
66];
67
68#[derive(Debug, Clone, PartialEq, Eq, Default)]
69pub struct PythonFiles {
70    /// Relative, `/`-separated paths of measured application sources.
71    pub sources: Vec<String>,
72    /// Relative paths of test modules, conftests and other excluded `.py`.
73    pub tests: Vec<String>,
74    pub dependency_files: Vec<PathBuf>,
75    pub configuration_files: Vec<PathBuf>,
76    pub excluded: Vec<(String, &'static str)>,
77}
78
79#[derive(Debug, Clone, PartialEq)]
80pub struct PreparedPythonProject {
81    pub root: PathBuf,
82    pub files: PythonFiles,
83    pub manifest: CoverageManifest,
84    pub plan: PythonProbePlan,
85    pub unparseable: Vec<(String, String)>,
86}
87
88fn is_venv(directory: &Path) -> bool {
89    fs::symlink_metadata(directory.join("pyvenv.cfg")).is_ok()
90}
91
92fn is_test_path(relative: &str) -> Option<&'static str> {
93    let mut components = relative.split('/').peekable();
94    let mut file_name = "";
95    while let Some(component) = components.next() {
96        if components.peek().is_none() {
97            file_name = component;
98            break;
99        }
100        if matches!(component, "tests" | "test" | "testing" | "__tests__") {
101            return Some("inside a test directory");
102        }
103    }
104    if file_name == "conftest.py" {
105        return Some("pytest conftest");
106    }
107    if file_name.starts_with("test_") && file_name.ends_with(".py") {
108        return Some("test module by name");
109    }
110    if file_name.ends_with("_test.py") || file_name.ends_with("_tests.py") {
111        return Some("test module by name");
112    }
113    if matches!(
114        file_name,
115        "setup.py" | "noxfile.py" | "tasks.py" | "fabfile.py"
116    ) {
117        return Some("build/test tooling script");
118    }
119    None
120}
121
122fn walk(
123    root: &Path,
124    directory: &Path,
125    files: &mut PythonFiles,
126    all_python: &mut Vec<String>,
127) -> Result<(), String> {
128    let mut entries = fs::read_dir(directory)
129        .map_err(|error| format!("{}: {error}", directory.display()))?
130        .collect::<Result<Vec<_>, _>>()
131        .map_err(|error| error.to_string())?;
132    entries.sort_by_key(fs::DirEntry::file_name);
133    for entry in entries {
134        let path = entry.path();
135        let name = entry.file_name().into_string().map_err(|_| {
136            format!(
137                "Python project contains a non-UTF-8 path: {}",
138                path.display()
139            )
140        })?;
141        let file_type = entry.file_type().map_err(|error| error.to_string())?;
142        let relative = path
143            .strip_prefix(root)
144            .map_err(|_| format!("path escaped root: {}", path.display()))?
145            .to_string_lossy()
146            .replace('\\', "/");
147        if file_type.is_dir() {
148            if (EXCLUDED_DIRECTORIES.contains(&name.as_str())
149                && (name != "target" || crate::source_discovery::target_is_build_output(&path)))
150                || name.ends_with(".egg-info")
151                || is_venv(&path)
152            {
153                files
154                    .excluded
155                    .push((relative, "tooling or environment directory"));
156                continue;
157            }
158            walk(root, &path, files, all_python)?;
159        } else if file_type.is_file() {
160            if DEPENDENCY_FILES.contains(&name.as_str())
161                || (name.starts_with("requirements") && name.ends_with(".txt"))
162            {
163                files.dependency_files.push(PathBuf::from(&relative));
164                if name != "setup.py" {
165                    continue;
166                }
167            }
168            // A type checker and a coverage tool do not change what the code
169            // does when it runs, so neither is execution context, any more than
170            // a formatter is. pytest and tox decide what executes;
171            // `.python-version` decides which interpreter executes it.
172            if matches!(name.as_str(), "pytest.ini" | "tox.ini" | ".python-version") {
173                files.configuration_files.push(PathBuf::from(&relative));
174                continue;
175            }
176            if !name.ends_with(".py") {
177                continue;
178            }
179            all_python.push(relative.clone());
180            match is_test_path(&relative) {
181                Some(reason) => {
182                    files.tests.push(relative.clone());
183                    files.excluded.push((relative, reason));
184                }
185                None => files.sources.push(relative),
186            }
187        }
188        // Symlinks are neither followed nor measured: a linked source tree is
189        // outside the project's own denominator.
190    }
191    Ok(())
192}
193
194pub fn discover_python_files(root: &Path) -> Result<PythonFiles, String> {
195    let mut files = PythonFiles::default();
196    let mut all_python = Vec::new();
197    walk(root, root, &mut files, &mut all_python)?;
198    files.sources.sort();
199    files.tests.sort();
200    files.dependency_files.sort();
201    files.configuration_files.sort();
202    Ok(files)
203}
204
205fn limitation(id: &str, kind: &str, file: &str, reason: &str) -> serde_json::Value {
206    json!({
207        "id": id,
208        "kind": kind,
209        "file": file,
210        "line": 1,
211        "column": 0,
212        "source": "",
213        "reason": reason
214    })
215}
216
217pub fn prepare_python_project(
218    root: &Path,
219    roots: Option<&crate::source_discovery::ExplicitSourceRoots>,
220) -> Result<PreparedPythonProject, String> {
221    let mut files = discover_python_files(root)?;
222    match roots {
223        Some(roots) => {
224            roots.narrow(&mut files.sources, &mut files.excluded, String::as_str);
225            roots.refuse_if_empty(files.sources.len(), "Python")?;
226        }
227        None => crate::source_discovery::leave_out_what_lies_beside(
228            &mut files.sources,
229            &mut files.excluded,
230        ),
231    }
232    if files.sources.is_empty() && files.tests.is_empty() {
233        return Err(
234            "no Python source files were found under the project root; Supercov measures .py files outside virtual environments, build output and test directories".into(),
235        );
236    }
237    let mut manifest = CoverageManifest {
238        unmeasured: Vec::new(),
239        decisions: Vec::new(),
240        points: Vec::new(),
241        branches: Vec::new(),
242        limitations: Vec::new(),
243        scope: None,
244        assertion_sites: Vec::new(),
245    };
246    let mut plan_files = BTreeMap::<String, PythonFilePlan>::new();
247    let mut limitation_ids = BTreeSet::new();
248    let mut unparseable = Vec::new();
249    for relative in &files.sources {
250        let path = root.join(relative);
251        let source = fs::read(&path).map_err(|error| format!("{}: {error}", path.display()))?;
252        let Ok(source) = String::from_utf8(source) else {
253            unparseable.push((relative.clone(), "source is not valid UTF-8".to_owned()));
254            continue;
255        };
256        match build_python_obligations(relative, &source) {
257            Ok(obligations) => {
258                manifest.points.extend(obligations.manifest.points);
259                manifest.decisions.extend(obligations.manifest.decisions);
260                manifest.branches.extend(obligations.manifest.branches);
261                manifest.unmeasured.extend(obligations.manifest.unmeasured);
262                for item in obligations.manifest.limitations {
263                    let id = item
264                        .get("id")
265                        .and_then(serde_json::Value::as_str)
266                        .unwrap_or_default()
267                        .to_owned();
268                    if limitation_ids.insert(id) {
269                        manifest.limitations.push(item);
270                    }
271                }
272                plan_files.insert(relative.clone(), obligations.plan);
273            }
274            Err(error) => unparseable.push((relative.clone(), error.to_string())),
275        }
276    }
277    if let Some((file, reason)) = unparseable.first()
278        && limitation_ids.insert(UNPARSEABLE_LIMITATION.into())
279    {
280        manifest.limitations.push(limitation(
281            UNPARSEABLE_LIMITATION,
282            "source-scope",
283            file,
284            &format!(
285                "{} source file(s) could not be parsed and carry no obligations; first: {file}: {reason}",
286                unparseable.len()
287            ),
288        ));
289    }
290    manifest.unmeasured.sort();
291    manifest.unmeasured.dedup();
292    let mut entries = Vec::new();
293    for file in &files.sources {
294        let unparseable_file = unparseable.iter().any(|(path, _)| path == file);
295        entries.push(SourceScopeEntry {
296            file: file.clone(),
297            status: if unparseable_file {
298                SourceScopeStatus::Excluded
299            } else {
300                SourceScopeStatus::Included
301            },
302            reason: if unparseable_file {
303                "could not be parsed".into()
304            } else {
305                "Python application source".into()
306            },
307            package_root: None,
308        });
309    }
310    for (file, reason) in &files.excluded {
311        if file.ends_with(".py") {
312            entries.push(SourceScopeEntry {
313                file: file.clone(),
314                status: SourceScopeStatus::Excluded,
315                reason: (*reason).into(),
316                package_root: None,
317            });
318        }
319    }
320    entries.sort_by(|left, right| left.file.cmp(&right.file));
321    manifest.scope = Some(
322        serde_json::to_value(SourceScope {
323            version: 1,
324            mode: if roots.is_some() {
325                SourceScopeMode::Explicit
326            } else {
327                SourceScopeMode::Automatic
328            },
329            roots: match roots {
330                Some(roots) => roots.local_roots().map_err(|error| error.to_string())?,
331                None => vec![".".into()],
332            },
333            entries,
334        })
335        .map_err(|error| error.to_string())?,
336    );
337    Ok(PreparedPythonProject {
338        root: root.to_owned(),
339        plan: PythonProbePlan {
340            version: PYTHON_PROBE_PLAN_VERSION,
341            root: root.display().to_string(),
342            files: plan_files,
343            assertion_sites: BTreeMap::new(),
344        },
345        manifest,
346        files,
347        unparseable,
348    })
349}
350
351/// Integrity inputs: sources and tests are hashed separately, dependency and
352/// configuration files identify the environment, and the test command
353/// identifies execution.
354///
355/// The ambient environment is deliberately absent. It is not a property of the
356/// project, it changes with every terminal, and folding it in made a run
357/// identity that no two machines could agree on. What a project actually needs
358/// from its environment is declared in the files above.
359pub fn python_integrity_inputs(files: &PythonFiles, command: &[String]) -> ExplicitIntegrityInputs {
360    let execution_configuration = command.join("\0").into_bytes();
361    ExplicitIntegrityInputs {
362        source_files: files.sources.iter().map(PathBuf::from).collect(),
363        test_files: files.tests.iter().map(PathBuf::from).collect(),
364        dependency_files: files.dependency_files.clone(),
365        configuration_files: files.configuration_files.clone(),
366        execution_configuration,
367    }
368}
369
370#[cfg(test)]
371mod tests {
372
373    #[test]
374    fn the_ambient_environment_is_not_part_of_run_identity() {
375        // Identity is what the project is, not which shell it was run from.
376        // Folding the environment in meant no two terminals, and no two
377        // machines, ever agreed on the same run. What a project genuinely needs
378        // from its environment it declares in the files that are hashed above.
379        let files = PythonFiles::default();
380        let command = ["pytest".to_owned(), "-q".to_owned()];
381        let inputs = python_integrity_inputs(&files, &command);
382        assert_eq!(inputs.execution_configuration, b"pytest\0-q");
383    }
384    use std::time::{SystemTime, UNIX_EPOCH};
385
386    use super::*;
387
388    fn fixture(name: &str) -> PathBuf {
389        let nonce = SystemTime::now()
390            .duration_since(UNIX_EPOCH)
391            .unwrap()
392            .as_nanos();
393        let root = std::env::temp_dir().join(format!(
394            "supercov-python-project-{}-{nonce}-{name}",
395            std::process::id()
396        ));
397        fs::create_dir_all(&root).unwrap();
398        root
399    }
400
401    fn write(root: &Path, relative: &str, contents: &str) {
402        let path = root.join(relative);
403        fs::create_dir_all(path.parent().unwrap()).unwrap();
404        fs::write(path, contents).unwrap();
405    }
406
407    #[test]
408    fn a_type_checker_and_a_coverage_tool_are_not_execution_context() {
409        // Neither changes what the code does when it runs, so neither should
410        // cost every flow in the map a re-reading. What decides execution does:
411        // pytest and tox choose what runs, `.python-version` chooses the
412        // interpreter that runs it.
413        let root = fixture("inert-tooling");
414        write(&root, "pyproject.toml", "[project]\nname='x'\n");
415        write(&root, "mypy.ini", "[mypy]\n");
416        write(&root, ".coveragerc", "[run]\n");
417        write(&root, "pytest.ini", "[pytest]\n");
418        write(&root, ".python-version", "3.13\n");
419        write(&root, "src/pkg/core.py", "def f(a):\n    return a\n");
420        let project = prepare_python_project(&root, None).unwrap();
421        assert_eq!(
422            project.files.configuration_files,
423            [
424                PathBuf::from(".python-version"),
425                PathBuf::from("pytest.ini")
426            ]
427        );
428        fs::remove_dir_all(root).unwrap();
429    }
430
431    fn scope_of(project: &PreparedPythonProject) -> crate::source_discovery::SourceScope {
432        serde_json::from_value(project.manifest.scope.clone().expect("a scope")).unwrap()
433    }
434
435    fn explicit(root: &Path, roots: &[&str]) -> crate::source_discovery::ExplicitSourceRoots {
436        let roots = roots
437            .iter()
438            .map(|root| (*root).to_owned())
439            .collect::<Vec<_>>();
440        crate::source_discovery::ExplicitSourceRoots::resolve(root, &roots).unwrap()
441    }
442
443    #[test]
444    fn explicit_roots_narrow_a_flat_layout_to_the_modules_it_names() {
445        // The reported project kept its own modules at the root with a vendored
446        // PySide6 tree beside them, and measured 660 files instead of about
447        // fifteen: nothing on the Python path read SUPERCOV_SOURCE_ROOTS. A flat
448        // layout has no directory to name, so a file has to be a root too, the
449        // way it already is for JavaScript.
450        let root = fixture("flat-roots");
451        write(&root, "a.py", "def f(x):\n    return x if x else 0\n");
452        write(&root, "vendor/pkg/b.py", "def g():\n    return 1\n");
453        write(&root, "test/test_a.py", "from a import f\n");
454
455        let project = prepare_python_project(&root, Some(&explicit(&root, &["a.py"]))).unwrap();
456        assert_eq!(project.files.sources, ["a.py"], "{:?}", project.files);
457        assert_eq!(
458            project.plan.files.keys().collect::<Vec<_>>(),
459            ["a.py"],
460            "the runtime is not handed a file the roots leave out, which is what \
461             keeps it from being instrumented at all"
462        );
463        assert!(
464            project
465                .files
466                .excluded
467                .contains(&("vendor/pkg/b.py".into(), "outside explicit source roots")),
468            "{:?}",
469            project.files.excluded
470        );
471        let scope = scope_of(&project);
472        assert_eq!(
473            scope.mode,
474            crate::source_discovery::SourceScopeMode::Explicit
475        );
476        assert_eq!(scope.roots, ["a.py"]);
477
478        // A directory is a root just as well, and narrows to what is under it.
479        let project = prepare_python_project(&root, Some(&explicit(&root, &["vendor"]))).unwrap();
480        assert_eq!(project.files.sources, ["vendor/pkg/b.py"]);
481
482        // Unset, discovery is what it always was.
483        let project = prepare_python_project(&root, None).unwrap();
484        assert_eq!(project.files.sources, ["a.py", "vendor/pkg/b.py"]);
485        assert_eq!(
486            scope_of(&project).mode,
487            crate::source_discovery::SourceScopeMode::Automatic
488        );
489        fs::remove_dir_all(root).ok();
490    }
491
492    #[test]
493    fn what_lies_beside_the_package_is_not_measured_unless_it_is_named() {
494        // h11 as it is laid out: the package, and beside it the files that
495        // build its documentation, show its use and time it. They were all
496        // source, none is run by a test, and the package's 99% read 69%.
497        let root = fixture("beside");
498        write(&root, "h11/_connection.py", "def send(x):\n    return x\n");
499        // A package of the product can be called `examples` too.
500        write(
501            &root,
502            "h11/examples/echo.py",
503            "def echo(x):\n    return x\n",
504        );
505        write(&root, "docs/source/conf.py", "project = 'h11'\n");
506        write(&root, "examples/basic-client.py", "import h11\n");
507        write(&root, "bench/benchmarks/benchmarks.py", "import h11\n");
508        write(&root, "fuzz/afl-server.py", "import h11\n");
509        write(&root, "h11/tests/test_connection.py", "import h11\n");
510
511        let project = prepare_python_project(&root, None).unwrap();
512        assert_eq!(
513            project.files.sources,
514            ["h11/_connection.py", "h11/examples/echo.py"],
515            "{:?}",
516            project.files
517        );
518        let scope = scope_of(&project);
519        let reason = |file: &str| {
520            let entry = scope
521                .entries
522                .iter()
523                .find(|entry| entry.file == file)
524                .unwrap();
525            assert_eq!(entry.status, SourceScopeStatus::Excluded, "{file}");
526            entry.reason.clone()
527        };
528        assert_eq!(reason("docs/source/conf.py"), "documentation");
529        assert_eq!(reason("examples/basic-client.py"), "example");
530        assert_eq!(reason("bench/benchmarks/benchmarks.py"), "benchmark");
531        assert_eq!(reason("fuzz/afl-server.py"), "fuzz harness");
532
533        // Whoever names `examples` as a root has said it is source.
534        let named =
535            prepare_python_project(&root, Some(&explicit(&root, &["examples", "h11"]))).unwrap();
536        assert_eq!(
537            named.files.sources,
538            [
539                "examples/basic-client.py",
540                "h11/_connection.py",
541                "h11/examples/echo.py"
542            ]
543        );
544        fs::remove_dir_all(root).ok();
545    }
546
547    #[test]
548    fn roots_that_name_nothing_are_refused_rather_than_measuring_nothing() {
549        // A typo in the variable must not become a green run over zero files,
550        // the failure mode most likely to pass unnoticed in an agent loop.
551        let root = fixture("roots-name-nothing");
552        write(&root, "a.py", "x = 1\n");
553        write(&root, "test/test_a.py", "import a\n");
554        let error = prepare_python_project(&root, Some(&explicit(&root, &["scr"]))).unwrap_err();
555        assert!(error.contains("SUPERCOV_SOURCE_ROOTS=scr"), "{error}");
556        assert!(
557            error.contains("scr does not exist"),
558            "the typo is what the person needs to see: {error}"
559        );
560        fs::remove_dir_all(root).ok();
561    }
562
563    #[test]
564    fn a_root_never_pulls_a_test_back_into_the_denominator() {
565        // Roots narrow what would be measured; they are not a second way to
566        // choose files. A root that happens to contain the test directory must
567        // still leave its tests out.
568        let root = fixture("roots-keep-tests-out");
569        write(&root, "a.py", "x = 1\n");
570        write(&root, "test/test_a.py", "import a\n");
571        let project = prepare_python_project(&root, Some(&explicit(&root, &["."]))).unwrap();
572        assert_eq!(project.files.sources, ["a.py"]);
573        assert!(project.files.tests.contains(&"test/test_a.py".to_owned()));
574        fs::remove_dir_all(root).ok();
575    }
576
577    #[test]
578    fn separates_sources_tests_environments_and_tooling() {
579        let root = fixture("discover");
580        write(&root, "pyproject.toml", "[project]\nname='x'\n");
581        write(&root, "pytest.ini", "[pytest]\n");
582        write(&root, "src/pkg/__init__.py", "");
583        write(&root, "src/pkg/core.py", "def f(a):\n    return a and 1\n");
584        write(&root, "tests/test_core.py", "def test():\n    pass\n");
585        write(&root, "conftest.py", "");
586        write(&root, "setup.py", "print(1)\n");
587        write(&root, ".venv/pyvenv.cfg", "home = /usr\n");
588        write(&root, ".venv/lib/site.py", "x = 1\n");
589        write(&root, "env2/pyvenv.cfg", "home = /usr\n");
590        write(&root, "env2/lib/thing.py", "y = 2\n");
591        write(&root, "broken/old.py", "print 'python 2'\n");
592        let project = prepare_python_project(&root, None).unwrap();
593        assert_eq!(
594            project.files.sources,
595            ["broken/old.py", "src/pkg/__init__.py", "src/pkg/core.py"]
596        );
597        assert_eq!(
598            project.files.tests,
599            ["conftest.py", "setup.py", "tests/test_core.py"]
600        );
601        assert_eq!(
602            project.files.dependency_files,
603            [PathBuf::from("pyproject.toml"), PathBuf::from("setup.py")]
604        );
605        assert_eq!(
606            project.files.configuration_files,
607            [PathBuf::from("pytest.ini")]
608        );
609        assert_eq!(project.unparseable.len(), 1);
610        assert!(project.plan.files.contains_key("src/pkg/core.py"));
611        assert!(!project.plan.files.contains_key("broken/old.py"));
612        let ids = project
613            .manifest
614            .limitations
615            .iter()
616            .map(|item| item["id"].as_str().unwrap().to_owned())
617            .collect::<BTreeSet<_>>();
618        assert_eq!(ids.len(), 1);
619        assert!(ids.contains(UNPARSEABLE_LIMITATION));
620        assert!(
621            project
622                .manifest
623                .points
624                .iter()
625                .all(|point| point.file.starts_with("src/"))
626        );
627        fs::remove_dir_all(root).unwrap();
628    }
629}
supercov_engine/python_project.rs

supercov_engine/
python_project.rs
