keel_cli/scan/
python.rs

1//! The Python static scan: an `ast`-walker executed out-of-process via
2//! `python3 -`.
3//!
4//! Parsing Python with Python's own `ast` is exact where a regex would guess:
5//! it sees real imports and string-literal constants with true line numbers.
6//! The walker script is embedded and fed on stdin; it prints one JSON object.
7//! If `python3` is absent the pass yields nothing and reports
8//! [`available`](PyScan::available)`= false` so the caller can say so out loud
9//! rather than silently under-reporting coverage.
10
11use std::io::Write;
12use std::path::Path;
13use std::process::{Command, Stdio};
14
15use serde::Deserialize;
16
17use super::{FunctionFacts, LangFindings, Sighting};
18
19/// The embedded `ast` walker. Deterministic: directories and files are visited
20/// in sorted order, output keys are sorted. Finds imports of the known effect
21/// libraries and URL/DSN string literals, each with `file:line` — and, for
22/// `keel flows suggest`, attributes effect / time / random / replay-unsafe
23/// calls to their enclosing **module-level** function defs (real AST
24/// containment: nested defs and lambdas inside a function count toward it;
25/// class methods are not flow entrypoints and are not attributed).
26const AST_WALKER: &str = r#"
27import ast, json, os, sys
28from urllib.parse import urlsplit
29
30HTTP_LIBS = {"httpx", "requests", "aiohttp", "urllib3"}
31LLM_LIBS = {"openai", "anthropic"}
32OTHER_LIBS = {"psycopg", "boto3"}
33KNOWN = HTTP_LIBS | LLM_LIBS | OTHER_LIBS
34TIME_LIBS = {"time", "datetime"}
35RANDOM_LIBS = {"random", "uuid", "secrets"}
36UNSAFE_LIBS = {"threading", "multiprocessing", "subprocess", "socket"}
37TRACKED = KNOWN | TIME_LIBS | RANDOM_LIBS | UNSAFE_LIBS | {"os"}
38TIME_NAMES = {"time", "time_ns", "monotonic", "monotonic_ns",
39              "perf_counter", "perf_counter_ns", "gmtime", "localtime"}
40DT_NAMES = {"now", "utcnow", "today"}
41UUID_NAMES = {"uuid1", "uuid3", "uuid4", "uuid5"}
42OS_UNSAFE = {"system", "popen", "fork", "forkpty", "execv", "execve",
43             "execvp", "execvpe", "spawnl", "spawnv", "spawnvp"}
44SKIP = {".keel", ".git", "__pycache__", "node_modules", ".venv", "venv",
45        ".mypy_cache", ".pytest_cache", "dist", "build", "target"}
46
47
48def top(mod):
49    return mod.split(".", 1)[0] if mod else ""
50
51
52def host(s):
53    if "://" not in s:
54        return None
55    try:
56        parts = urlsplit(s.strip())
57    except ValueError:
58        return None
59    if not parts.scheme or not parts.hostname:
60        return None
61    return parts.hostname
62
63
64def call_root(f):
65    """The Name at the base of a call's attribute chain, or None. Deliberately
66    does NOT see through intermediate calls: in `httpx.get(u).json()` only the
67    inner `httpx.get(u)` has a root, so a chained method on a call result is
68    never double-counted as a second effect."""
69    while isinstance(f, ast.Attribute):
70        f = f.value
71    return f.id if isinstance(f, ast.Name) else None
72
73
74# Call names that construct a handle rather than perform an effect: CapWords
75# constructors (OpenAI(), Client()) plus the well-known factory methods.
76FACTORY_NAMES = {"client", "resource", "session", "connect"}
77
78
79def is_constructor(name):
80    return name is None or name[:1].isupper() or name in FACTORY_NAMES
81
82
83def aliases_of(tree):
84    """Binding name -> tracked top-level module. Also follows one hop of
85    constructor assignment (client = OpenAI() -> client is an openai handle),
86    the dominant SDK-client pattern."""
87    a = {}
88    for node in ast.walk(tree):
89        if isinstance(node, ast.Import):
90            for al in node.names:
91                t = top(al.name)
92                if t in TRACKED:
93                    a[(al.asname or al.name).split(".", 1)[0]] = t
94        elif isinstance(node, ast.ImportFrom):
95            t = top(node.module or "")
96            if t in TRACKED:
97                for al in node.names:
98                    a[al.asname or al.name] = t
99    for node in ast.walk(tree):
100        if isinstance(node, ast.Assign) and isinstance(node.value, ast.Call):
101            lib = a.get(call_root(node.value.func))
102            if lib in KNOWN:
103                for tgt in node.targets:
104                    if isinstance(tgt, ast.Name):
105                        a[tgt.id] = lib
106    return a
107
108
109def url_consts_of(tree):
110    """Module-level NAME = "scheme://host/..." constants -> host, so a URL
111    hoisted to a constant still attributes to the functions that use it."""
112    consts = {}
113    for node in tree.body:
114        if (isinstance(node, ast.Assign) and isinstance(node.value, ast.Constant)
115                and isinstance(node.value.value, str)):
116            h = host(node.value.value)
117            if h:
118                for tgt in node.targets:
119                    if isinstance(tgt, ast.Name):
120                        consts[tgt.id] = h
121    return consts
122
123
124def fn_facts(fn, rel, mod, aliases, url_consts):
125    effects = unsafe_idem = t_reads = r_reads = 0
126    targets = set()
127    reasons = []
128    for node in ast.walk(fn):
129        if isinstance(node, ast.Name) and node.id in url_consts:
130            targets.add(url_consts[node.id])
131        elif isinstance(node, ast.Constant) and isinstance(node.value, str):
132            h = host(node.value)
133            if h:
134                targets.add(h)
135        if not isinstance(node, ast.Call):
136            continue
137        lib = aliases.get(call_root(node.func))
138        if lib is None:
139            continue
140        attr = node.func.attr if isinstance(node.func, ast.Attribute) else None
141        name = attr if attr is not None else call_root(node.func)
142        if lib in KNOWN:
143            if is_constructor(name):
144                continue  # a handle being built, not an effect performed
145            effects += 1
146            if name in {"post", "patch"}:
147                unsafe_idem += 1
148            if lib in LLM_LIBS:
149                targets.add("llm:" + lib)
150        elif lib == "time" and name in TIME_NAMES:
151            t_reads += 1
152        elif lib == "datetime" and name in DT_NAMES:
153            t_reads += 1
154        elif lib in {"random", "secrets"}:
155            r_reads += 1
156        elif lib == "uuid" and name in UUID_NAMES:
157            r_reads += 1
158        elif lib == "os" and name == "urandom":
159            r_reads += 1
160        elif lib in UNSAFE_LIBS:
161            reasons.append((node.lineno, "%s use at %s:%d" % (lib, rel, node.lineno)))
162        elif lib == "os" and name in OS_UNSAFE:
163            reasons.append((node.lineno, "os.%s at %s:%d" % (name, rel, node.lineno)))
164    return {"effects": effects, "file": rel, "idempotent_unsafe": unsafe_idem,
165            "line": fn.lineno, "module": mod, "name": fn.name,
166            "random_reads": r_reads, "targets": sorted(targets),
167            "time_reads": t_reads,
168            "unsafe_reasons": [t for _, t in sorted(reasons)]}
169
170
171root = sys.argv[1] if len(sys.argv) > 1 else "."
172imports = []
173urls = []
174functions = []
175files = 0
176for dirpath, dirnames, filenames in os.walk(root):
177    dirnames[:] = sorted(d for d in dirnames if d not in SKIP and not d.startswith("."))
178    for fn in sorted(filenames):
179        if not fn.endswith(".py"):
180            continue
181        path = os.path.join(dirpath, fn)
182        rel = os.path.relpath(path, root).replace(os.sep, "/")
183        try:
184            with open(path, "r", encoding="utf-8") as fh:
185                tree = ast.parse(fh.read())
186        except (OSError, SyntaxError, UnicodeDecodeError, ValueError):
187            continue
188        files += 1
189        for node in ast.walk(tree):
190            if isinstance(node, ast.Import):
191                for alias in node.names:
192                    t = top(alias.name)
193                    if t in KNOWN:
194                        imports.append({"lib": t, "file": rel, "line": node.lineno})
195            elif isinstance(node, ast.ImportFrom):
196                t = top(node.module or "")
197                if t in KNOWN:
198                    imports.append({"lib": t, "file": rel, "line": node.lineno})
199            elif isinstance(node, ast.Constant) and isinstance(node.value, str):
200                h = host(node.value)
201                if h:
202                    urls.append({"host": h, "file": rel, "line": node.lineno})
203        mod = rel[:-3].replace("/", ".")
204        if mod.endswith(".__init__"):
205            mod = mod[: -len(".__init__")]
206        aliases = aliases_of(tree)
207        consts = url_consts_of(tree)
208        for node in tree.body:
209            if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
210                functions.append(fn_facts(node, rel, mod, aliases, consts))
211
212print(json.dumps({"files_scanned": files, "functions": functions,
213                  "imports": imports, "urls": urls}, sort_keys=True))
214"#;
215
216/// One import finding from the walker.
217#[derive(Debug, Deserialize)]
218struct Import {
219    lib: String,
220    file: String,
221    line: u32,
222}
223
224/// One URL-literal finding from the walker.
225#[derive(Debug, Deserialize)]
226struct Url {
227    host: String,
228    file: String,
229    line: u32,
230}
231
232/// One module-level function's facts from the walker.
233#[derive(Debug, Deserialize)]
234struct PyFunction {
235    effects: u32,
236    file: String,
237    idempotent_unsafe: u32,
238    line: u32,
239    module: String,
240    name: String,
241    random_reads: u32,
242    targets: Vec<String>,
243    time_reads: u32,
244    unsafe_reasons: Vec<String>,
245}
246
247/// The walker's JSON output, typed.
248#[derive(Debug, Deserialize)]
249struct WalkerOutput {
250    files_scanned: usize,
251    #[serde(default)]
252    functions: Vec<PyFunction>,
253    imports: Vec<Import>,
254    urls: Vec<Url>,
255}
256
257/// The Python pass result.
258#[derive(Debug, Clone, Default)]
259pub struct PyScan {
260    /// Whether `python3` ran the walker.
261    pub available: bool,
262    /// Files the walker parsed.
263    pub files_scanned: usize,
264    /// Findings, ready to merge.
265    pub findings: LangFindings,
266    /// Per-function attribution (module-level defs), for `keel flows suggest`.
267    pub functions: Vec<FunctionFacts>,
268}
269
270const HTTP_LIBS: &[&str] = &["httpx", "requests", "aiohttp", "urllib3"];
271
272/// Run the walker over `project`. A missing `python3`, or a walker that fails,
273/// yields an empty unavailable result — never a panic.
274pub fn scan(project: &Path) -> PyScan {
275    let Some(output) = run_walker(project) else {
276        return PyScan::default();
277    };
278    let mut findings = LangFindings::default();
279    for imp in &output.imports {
280        findings.libs.insert(imp.lib.clone());
281        let sighting = Sighting {
282            file: imp.file.clone(),
283            line: imp.line,
284        };
285        match imp.lib.as_str() {
286            "openai" => findings.llm.push(("openai".to_owned(), sighting)),
287            "anthropic" => findings.llm.push(("anthropic".to_owned(), sighting)),
288            lib if HTTP_LIBS.contains(&lib) => findings.http_in_use = true,
289            // psycopg/boto3: recorded as effect libraries via their DSN/URL
290            // literals (if any); no synthetic host target from the import alone.
291            _ => {}
292        }
293    }
294    // A DSN literal (postgres://…) is itself evidence of an outbound call even
295    // without one of the HTTP libraries imported.
296    if !output.urls.is_empty() {
297        findings.http_in_use = true;
298    }
299    for url in &output.urls {
300        // The walker already returned a bare hostname (urlsplit.hostname), so it
301        // is lowercased and port-stripped; normalize defensively.
302        findings.hosts.push((
303            url.host.to_ascii_lowercase(),
304            Sighting {
305                file: url.file.clone(),
306                line: url.line,
307            },
308        ));
309    }
310    let functions = output
311        .functions
312        .into_iter()
313        .map(|f| FunctionFacts {
314            entrypoint: format!("py:{}:{}", f.module, f.name),
315            file: f.file,
316            line: f.line,
317            effects: f.effects,
318            idempotent_unsafe: f.idempotent_unsafe,
319            time_reads: f.time_reads,
320            random_reads: f.random_reads,
321            unsafe_reasons: f.unsafe_reasons,
322            targets: f.targets.into_iter().collect(),
323        })
324        .collect();
325    PyScan {
326        available: true,
327        files_scanned: output.files_scanned,
328        findings,
329        functions,
330    }
331}
332
333/// Spawn `python3 - <root>`, feed the walker on stdin, parse its stdout.
334fn run_walker(project: &Path) -> Option<WalkerOutput> {
335    let mut child = Command::new("python3")
336        .arg("-")
337        .arg(project)
338        .stdin(Stdio::piped())
339        .stdout(Stdio::piped())
340        .stderr(Stdio::null())
341        .spawn()
342        .ok()?;
343    child.stdin.take()?.write_all(AST_WALKER.as_bytes()).ok()?;
344    let out = child.wait_with_output().ok()?;
345    if !out.status.success() {
346        return None;
347    }
348    serde_json::from_slice(&out.stdout).ok()
349}
350
351#[cfg(test)]
352mod tests {
353    use super::*;
354    use std::fs;
355    use tempfile::TempDir;
356
357    fn python3_present() -> bool {
358        Command::new("python3")
359            .arg("--version")
360            .stdout(Stdio::null())
361            .stderr(Stdio::null())
362            .status()
363            .is_ok_and(|s| s.success())
364    }
365
366    #[test]
367    fn walks_imports_and_url_literals() {
368        if !python3_present() {
369            eprintln!("skip: python3 not available");
370            return;
371        }
372        let dir = TempDir::new().unwrap();
373        fs::write(
374            dir.path().join("app.py"),
375            "import httpx\nfrom openai import OpenAI\n\nURL = \"https://api.example.com/v1\"\n",
376        )
377        .unwrap();
378        let scan = scan(dir.path());
379        assert!(scan.available);
380        assert_eq!(scan.files_scanned, 1);
381        assert!(scan.findings.http_in_use);
382        assert!(
383            scan.findings
384                .llm
385                .iter()
386                .any(|(p, s)| p == "openai" && s.file == "app.py" && s.line == 2)
387        );
388        assert!(
389            scan.findings
390                .hosts
391                .iter()
392                .any(|(h, s)| h == "api.example.com" && s.line == 4)
393        );
394    }
395
396    #[test]
397    fn attributes_effects_time_random_to_module_level_functions() {
398        if !python3_present() {
399            eprintln!("skip: python3 not available");
400            return;
401        }
402        let dir = TempDir::new().unwrap();
403        fs::write(
404            dir.path().join("pipeline.py"),
405            r#"import time
406import random
407import httpx
408from openai import OpenAI
409
410API = "https://api.example.com/v1/data"
411client = OpenAI()
412
413
414def main():
415    started = time.time()
416    seed = random.random()
417    data = httpx.get(API).json()
418    httpx.post(API, json=data)
419    client.responses.create(model="gpt-4.1", input="hi")
420    return started, seed
421
422
423def helper():
424    return 41 + 1
425"#,
426        )
427        .unwrap();
428        let s = scan(dir.path());
429        let main = s
430            .functions
431            .iter()
432            .find(|f| f.entrypoint == "py:pipeline:main")
433            .expect("main attributed");
434        // get + post + create — the chained .json() must NOT double-count.
435        assert_eq!(main.effects, 3);
436        assert_eq!(main.idempotent_unsafe, 1, "only the POST");
437        assert_eq!(main.time_reads, 1);
438        assert_eq!(main.random_reads, 1);
439        assert!(main.unsafe_reasons.is_empty());
440        assert!(main.targets.contains("api.example.com"), "URL via constant");
441        assert!(main.targets.contains("llm:openai"), "client = OpenAI() hop");
442        assert_eq!((main.file.as_str(), main.line), ("pipeline.py", 10));
443        let helper = s
444            .functions
445            .iter()
446            .find(|f| f.entrypoint == "py:pipeline:helper")
447            .expect("helper attributed");
448        assert_eq!(helper.effects, 0);
449    }
450
451    #[test]
452    fn threads_and_subprocess_defeat_the_replay_safe_estimate() {
453        if !python3_present() {
454            eprintln!("skip: python3 not available");
455            return;
456        }
457        let dir = TempDir::new().unwrap();
458        fs::write(
459            dir.path().join("jobs.py"),
460            r#"import subprocess
461import threading
462import requests
463
464
465def risky():
466    requests.post("https://api.example.com/v1/x")
467    threading.Thread(target=print).start()
468    subprocess.run(["ls"])
469"#,
470        )
471        .unwrap();
472        let s = scan(dir.path());
473        let f = s
474            .functions
475            .iter()
476            .find(|f| f.entrypoint == "py:jobs:risky")
477            .expect("risky attributed");
478        assert_eq!(f.effects, 1);
479        assert_eq!(
480            f.unsafe_reasons,
481            vec![
482                "threading use at jobs.py:8".to_owned(),
483                "subprocess use at jobs.py:9".to_owned(),
484            ]
485        );
486    }
487
488    #[test]
489    fn syntax_error_file_is_skipped_not_fatal() {
490        if !python3_present() {
491            eprintln!("skip: python3 not available");
492            return;
493        }
494        let dir = TempDir::new().unwrap();
495        fs::write(dir.path().join("broken.py"), "def (:\n").unwrap();
496        fs::write(dir.path().join("ok.py"), "import requests\n").unwrap();
497        let scan = scan(dir.path());
498        assert_eq!(scan.files_scanned, 1, "only the parseable file counts");
499        assert!(scan.findings.http_in_use);
500    }
501}
keel_cli/scan/python.rs

keel_cli/scan/
python.rs
