Skip to main content

reflex/parsers/api/
python.rs

1//! Python API extraction by walking the syntax tree.
2//!
3//! Statements are read in source order; function bodies are never entered. What each
4//! statement becomes:
5//!
6//! | Source | Item |
7//! | --- | --- |
8//! | `def f(…)` / `async def f(…)` | `Function` (`Method` in a class) |
9//! | `class C(Base)` | `Class`; `Enum` when a base is an `…Enum` / `…Flag` |
10//! | `@property` / `@cached_property` | `Field` (setters and deleters are folded away) |
11//! | `x: int = 0` in a class body, `X = 0` | `Field` (`Variant` in an enum) |
12//! | `MAX = 3`, `x: int = 3` at module level | `Const` |
13//! | `X: TypeAlias = …`, `type X = …`, `X = NewType(…)`, `X = Union[…]` | `TypeAlias` |
14//! | `__all__ = [...]` (also `+=`, `.extend`, `.append`) | [`ApiFile::exports`]; `None` when any part is computed |
15//! | `from .x import A as B`, `from pkg import *` | [`ReExport`] (`.x.A` → `B`) |
16//!
17//! Docstrings (the first statement of a module, class or function; a string right after
18//! an assignment; `#:` comments before one) go through [`super::pydoc`]. Decorators are
19//! kept in `attrs` without the `@`. `@deprecated(…)` (PEP 702, `typing_extensions`,
20//! the `deprecation` package) and `.. deprecated::` set `deprecated`.
21//!
22//! Visibility: a leading `_` is `Private`; a dunder (`__init__`) is `Inherited`, and
23//! dunders are `hidden` except the ones a caller uses directly: `__init__` (documented as
24//! the constructor), `__call__`, and the context-manager, iterator, container and
25//! awaitable protocols. `__repr__`, `__eq__`, `__hash__` and friends are noise on a
26//! reference page. `@typing.overload` stubs are dropped when an implementation follows.
27//!
28//! Top-level `if`/`try` blocks are walked (first branch only), except
29//! `if __name__ == "__main__":`, so version-gated and optional-import definitions count.
30
31use super::collapse_ws;
32use super::pydoc::{self, Entry, PyDoc};
33use super::{ApiFile, ApiItem, ApiKind, Deprecation, Param, ReExport, SigParts, Visibility};
34use crate::models::Language;
35use anyhow::{Context, Result};
36use regex::Regex;
37use std::sync::LazyLock;
38use tree_sitter::Node;
39
40/// Longest signature kept, in characters.
41const MAX_SIGNATURE: usize = 600;
42/// Longest value shown in a constant's or field's signature.
43const MAX_VALUE: usize = 60;
44
45/// Dunder methods documented on a class page.
46const SHOWN_DUNDERS: &[&str] = &[
47    "__init__",
48    "__call__",
49    "__enter__",
50    "__exit__",
51    "__aenter__",
52    "__aexit__",
53    "__iter__",
54    "__aiter__",
55    "__next__",
56    "__anext__",
57    "__await__",
58    "__getitem__",
59    "__setitem__",
60    "__delitem__",
61    "__contains__",
62    "__len__",
63];
64
65pub fn extract(source: &str) -> Result<ApiFile> {
66    let mut parser = tree_sitter::Parser::new();
67    parser
68        .set_language(&crate::parsers::ParserFactory::get_language_grammar(
69            Language::Python,
70        )?)
71        .context("loading the Python grammar")?;
72    let tree = parser.parse(source, None).context("parsing Python")?;
73    let root = tree.root_node();
74
75    let mut w = Walker {
76        src: source,
77        reexports: Vec::new(),
78        exports: None,
79        exports_dynamic: false,
80    };
81    let (doc, items) = w.body(&root, Scope::Module);
82    Ok(ApiFile {
83        module_doc: doc.and_then(|d| d.doc),
84        items,
85        mod_decls: Vec::new(),
86        reexports: w.reexports,
87        // A list built from other lists (`__all__ = base.__all__ + [...]`) is unknown:
88        // the leading-underscore rule applies instead.
89        exports: w.exports.filter(|_| !w.exports_dynamic),
90        package: None,
91    })
92}
93
94#[derive(Clone, Copy, PartialEq)]
95enum Scope {
96    Module,
97    Class { is_enum: bool },
98}
99
100struct Walker<'a> {
101    src: &'a str,
102    reexports: Vec<ReExport>,
103    exports: Option<Vec<String>>,
104    /// `__all__` has a part that is not a string literal.
105    exports_dynamic: bool,
106}
107
108fn text<'s>(node: &Node, src: &'s str) -> &'s str {
109    node.utf8_text(src.as_bytes()).unwrap_or("")
110}
111
112fn line(node: &Node) -> u32 {
113    node.start_position().row as u32 + 1
114}
115
116fn end_line(node: &Node) -> u32 {
117    node.end_position().row as u32 + 1
118}
119
120fn is_dunder(name: &str) -> bool {
121    name.len() > 4 && name.starts_with("__") && name.ends_with("__")
122}
123
124fn visibility(name: &str) -> Visibility {
125    if is_dunder(name) {
126        Visibility::Inherited
127    } else if name.starts_with('_') {
128        Visibility::Private
129    } else {
130        Visibility::Public
131    }
132}
133
134static UPPER_RE: LazyLock<Regex> =
135    LazyLock::new(|| Regex::new(r"^_*[A-Z][A-Z0-9_]*$").expect("valid regex"));
136static CAPWORDS_RE: LazyLock<Regex> =
137    LazyLock::new(|| Regex::new(r"^_*[A-Z][A-Za-z0-9]*[a-z][A-Za-z0-9]*$").expect("valid regex"));
138
139/// Last dotted segment of a decorator's callee: `functools.cached_property` → `cached_property`.
140fn decorator_name(d: &str) -> &str {
141    let callee = d.split('(').next().unwrap_or(d).trim();
142    callee.rsplit('.').next().unwrap_or(callee)
143}
144
145fn has_decorator(attrs: &[String], names: &[&str]) -> bool {
146    attrs.iter().any(|a| names.contains(&decorator_name(a)))
147}
148
149static DEP_POSITIONAL_RE: LazyLock<Regex> = LazyLock::new(|| {
150    Regex::new(r#"^[\w.]+\(\s*[rRuU]?(?:"([^"]*)"|'([^']*)')"#).expect("valid regex")
151});
152static DEP_SINCE_RE: LazyLock<Regex> = LazyLock::new(|| {
153    Regex::new(r#"\b(?:version|since|deprecated_in)\s*=\s*["']([^"']*)["']"#).expect("valid regex")
154});
155static DEP_NOTE_RE: LazyLock<Regex> = LazyLock::new(|| {
156    Regex::new(r#"\b(?:reason|details|message|msg)\s*=\s*["']([^"']*)["']"#).expect("valid regex")
157});
158
159/// `@deprecated("msg")`, `@typing_extensions.deprecated(…)`, `@deprecated(deprecated_in=…, details=…)`.
160fn deprecation(attrs: &[String]) -> Option<Deprecation> {
161    let d = attrs.iter().find(|a| decorator_name(a) == "deprecated")?;
162    let cap = |re: &Regex| {
163        re.captures(d)
164            .and_then(|c| c.get(1).or_else(|| c.get(2)))
165            .map(|m| m.as_str().to_string())
166    };
167    Some(Deprecation {
168        since: cap(&DEP_SINCE_RE),
169        note: cap(&DEP_POSITIONAL_RE).or_else(|| cap(&DEP_NOTE_RE)),
170    })
171}
172
173/// `value` collapsed and cut to [`MAX_VALUE`] characters.
174fn short_value(v: &str) -> String {
175    let v = collapse_ws(v);
176    if v.chars().count() > MAX_VALUE {
177        v.chars().take(MAX_VALUE - 1).collect::<String>() + "…"
178    } else {
179        v
180    }
181}
182
183/// Declaration text from `node` up to its body, without the trailing `:`.
184fn signature(node: &Node, src: &str) -> String {
185    let end = node
186        .child_by_field_name("body")
187        .map(|b| b.start_byte())
188        .unwrap_or(node.end_byte());
189    let raw = &src[node.start_byte()..end];
190    let raw = raw.trim_end();
191    let raw = raw.strip_suffix(':').unwrap_or(raw);
192    let mut s = collapse_ws(raw);
193    if s.chars().count() > MAX_SIGNATURE {
194        s = s.chars().take(MAX_SIGNATURE).collect::<String>() + " …";
195    }
196    s
197}
198
199/// The docstring of a block: its first statement, when that is a string.
200fn docstring(block: &Node, src: &str) -> Option<PyDoc> {
201    let mut c = block.walk();
202    let first = block
203        .named_children(&mut c)
204        .find(|n| n.kind() != "comment")?;
205    string_statement(&first, src)
206}
207
208/// A statement that is a bare string literal, parsed as a docstring.
209fn string_statement(stmt: &Node, src: &str) -> Option<PyDoc> {
210    if stmt.kind() != "expression_statement" || stmt.named_child_count() != 1 {
211        return None;
212    }
213    let s = stmt.named_child(0)?;
214    (s.kind() == "string").then(|| pydoc::parse(text(&s, src), line(&s), end_line(&s)))
215}
216
217/// String literals directly inside a list / tuple (and `+` of them). Returns `false`
218/// when some part is not a literal (`other.__all__`, a name, a comprehension).
219fn string_list(node: &Node, src: &str, out: &mut Vec<String>) -> bool {
220    match node.kind() {
221        "string" => match pydoc::literal_value(text(node, src)) {
222            Some(v) => {
223                out.push(v);
224                true
225            }
226            None => false,
227        },
228        "list" | "tuple" | "binary_operator" | "parenthesized_expression" | "argument_list" => {
229            let mut c = node.walk();
230            let mut complete = true;
231            for ch in node.named_children(&mut c) {
232                complete &= string_list(&ch, src, out);
233            }
234            complete
235        }
236        "comment" => true,
237        _ => false,
238    }
239}
240
241impl Walker<'_> {
242    /// `__all__ = …` (`replace`) or `__all__ += …` / `.extend(…)` / `.append(…)`.
243    fn add_exports(&mut self, value: &Node, replace: bool) {
244        let mut names = Vec::new();
245        let complete = string_list(value, self.src, &mut names);
246        if replace {
247            self.exports = Some(names);
248            self.exports_dynamic = !complete;
249        } else {
250            self.exports.get_or_insert_default().extend(names);
251            self.exports_dynamic |= !complete;
252        }
253    }
254
255    /// A module or class body: its docstring and items.
256    fn body(&mut self, block: &Node, scope: Scope) -> (Option<PyDoc>, Vec<ApiItem>) {
257        let doc = docstring(block, self.src);
258        let mut out = Vec::new();
259        self.statements(block, scope, doc.is_some(), &mut out);
260        drop_overloads(&mut out);
261        (doc, out)
262    }
263
264    fn statements(&mut self, block: &Node, scope: Scope, skip_first: bool, out: &mut Vec<ApiItem>) {
265        let src = self.src;
266        let mut c = block.walk();
267        let children: Vec<Node> = block.named_children(&mut c).collect();
268        let mut skipped = !skip_first;
269        // `#:` comment lines before the next assignment.
270        let mut comment_doc: Vec<(String, u32)> = Vec::new();
271        // Index in `out` of an item that a following string literal documents.
272        let mut last_assign: Option<usize> = None;
273        for child in &children {
274            if child.kind() == "comment" {
275                match text(child, src).strip_prefix("#:") {
276                    Some(t) => comment_doc
277                        .push((t.strip_prefix(' ').unwrap_or(t).to_string(), line(child))),
278                    None => comment_doc.clear(),
279                }
280                continue;
281            }
282            if !skipped {
283                skipped = true;
284                continue;
285            }
286            let assign = last_assign.take();
287            let comments = std::mem::take(&mut comment_doc);
288            match child.kind() {
289                "function_definition" => {
290                    if let Some(it) = self.function(child, child, Vec::new(), scope) {
291                        out.push(it);
292                    }
293                }
294                "class_definition" => out.push(self.class(child, child, Vec::new(), scope)),
295                "decorated_definition" => {
296                    let mut dc = child.walk();
297                    let attrs: Vec<String> = child
298                        .named_children(&mut dc)
299                        .filter(|n| n.kind() == "decorator")
300                        .map(|n| collapse_ws(text(&n, src).trim_start_matches('@').trim()))
301                        .collect();
302                    match child.child_by_field_name("definition") {
303                        Some(d) if d.kind() == "function_definition" => {
304                            if let Some(it) = self.function(&d, child, attrs, scope) {
305                                out.push(it);
306                            }
307                        }
308                        Some(d) if d.kind() == "class_definition" => {
309                            out.push(self.class(&d, child, attrs, scope));
310                        }
311                        _ => {}
312                    }
313                }
314                "expression_statement" => {
315                    let Some(inner) = child.named_child(0) else {
316                        continue;
317                    };
318                    match inner.kind() {
319                        "assignment" => {
320                            if let Some(mut it) = self.assignment(&inner, child, scope) {
321                                if !comments.is_empty() {
322                                    let lines: Vec<&str> =
323                                        comments.iter().map(|(t, _)| t.as_str()).collect();
324                                    let d = pydoc::from_text(
325                                        &lines.join("\n"),
326                                        comments[0].1,
327                                        comments[comments.len() - 1].1,
328                                    );
329                                    it.doc = d.doc;
330                                }
331                                out.push(it);
332                                last_assign = Some(out.len() - 1);
333                            }
334                        }
335                        "string" => {
336                            // An attribute docstring: a string right after an assignment.
337                            if let Some(i) = assign
338                                && out[i].doc.is_none()
339                                && let Some(d) = string_statement(child, src)
340                            {
341                                out[i].doc = d.doc;
342                            }
343                        }
344                        "augmented_assignment" if scope == Scope::Module => {
345                            let left = inner.child_by_field_name("left");
346                            if left.is_some_and(|l| text(&l, src) == "__all__")
347                                && let Some(r) = inner.child_by_field_name("right")
348                            {
349                                self.add_exports(&r, false);
350                            }
351                        }
352                        "call" if scope == Scope::Module => {
353                            let f = inner
354                                .child_by_field_name("function")
355                                .map(|f| text(&f, src))
356                                .unwrap_or("");
357                            if (f == "__all__.extend" || f == "__all__.append")
358                                && let Some(args) = inner.child_by_field_name("arguments")
359                            {
360                                self.add_exports(&args, false);
361                            }
362                        }
363                        _ => {}
364                    }
365                }
366                "type_alias_statement" => {
367                    let name = child
368                        .child_by_field_name("left")
369                        .map(|l| {
370                            let t = text(&l, src);
371                            t.split(['[', ' ']).next().unwrap_or(t).to_string()
372                        })
373                        .unwrap_or_default();
374                    let mut it = base(child, ApiKind::TypeAlias, name);
375                    it.signature = short_signature(text(child, src));
376                    out.push(it);
377                }
378                "import_from_statement" if scope == Scope::Module => self.import_from(child),
379                "if_statement" if scope == Scope::Module => {
380                    let cond = child
381                        .child_by_field_name("condition")
382                        .map(|c| text(&c, src))
383                        .unwrap_or("");
384                    if !cond.contains("__name__")
385                        && let Some(b) = child.child_by_field_name("consequence")
386                    {
387                        self.statements(&b, scope, false, out);
388                    }
389                }
390                "try_statement" if scope == Scope::Module => {
391                    if let Some(b) = child.child_by_field_name("body") {
392                        self.statements(&b, scope, false, out);
393                    }
394                }
395                _ => {}
396            }
397        }
398    }
399
400    fn import_from(&mut self, node: &Node) {
401        let src = self.src;
402        let module = node
403            .child_by_field_name("module_name")
404            .map(|m| collapse_ws(text(&m, src)).replace(' ', ""))
405            .unwrap_or_default();
406        if module == "__future__" {
407            return;
408        }
409        let join = |name: &str| {
410            if module.ends_with('.') {
411                format!("{module}{name}")
412            } else {
413                format!("{module}.{name}")
414            }
415        };
416        let mut c = node.walk();
417        let mut push = |path: String, name: String| {
418            self.reexports.push(ReExport {
419                path,
420                visibility: visibility(&name),
421                name,
422                line: line(node),
423            });
424        };
425        for ch in node.children_by_field_name("name", &mut c) {
426            match ch.kind() {
427                "dotted_name" => {
428                    let n = text(&ch, src).to_string();
429                    push(join(&n), n);
430                }
431                "aliased_import" => {
432                    let n = ch
433                        .child_by_field_name("name")
434                        .map(|n| text(&n, src).to_string())
435                        .unwrap_or_default();
436                    let alias = ch
437                        .child_by_field_name("alias")
438                        .map(|a| text(&a, src).to_string())
439                        .unwrap_or_else(|| n.clone());
440                    push(join(&n), alias);
441                }
442                _ => {}
443            }
444        }
445        let mut c = node.walk();
446        if node
447            .named_children(&mut c)
448            .any(|n| n.kind() == "wildcard_import")
449        {
450            push(join("*"), "*".into());
451        }
452    }
453
454    fn function(
455        &mut self,
456        def: &Node,
457        outer: &Node,
458        attrs: Vec<String>,
459        scope: Scope,
460    ) -> Option<ApiItem> {
461        let src = self.src;
462        let name = def
463            .child_by_field_name("name")
464            .map(|n| text(&n, src).to_string())
465            .unwrap_or_default();
466        // `@x.setter` / `@x.deleter` belong to the property `x`.
467        if attrs.iter().any(|a| {
468            a.split('(')
469                .next()
470                .is_some_and(|c| c.ends_with(".setter") || c.ends_with(".deleter"))
471        }) {
472            return None;
473        }
474        let in_class = matches!(scope, Scope::Class { .. });
475        let is_property =
476            in_class && has_decorator(&attrs, &["property", "cached_property", "abstractproperty"]);
477        let is_static = has_decorator(&attrs, &["staticmethod"]);
478        let pydoc = def
479            .child_by_field_name("body")
480            .and_then(|b| docstring(&b, src))
481            .unwrap_or_default();
482
483        let mut sig = SigParts {
484            is_async: def.child(0).is_some_and(|c| c.kind() == "async"),
485            ..SigParts::default()
486        };
487        if let Some(tp) = def.child_by_field_name("type_parameters") {
488            sig.generics = Some(collapse_ws(text(&tp, src)));
489        }
490        if let Some(r) = def.child_by_field_name("return_type") {
491            sig.returns = Some(collapse_ws(text(&r, src)));
492        }
493        if let Some(ps) = def.child_by_field_name("parameters") {
494            sig.params = params(&ps, src);
495        }
496        if in_class && !is_static && !sig.params.is_empty() && !sig.params[0].name.starts_with('*')
497        {
498            sig.receiver = Some(sig.params.remove(0).name);
499        }
500        fill_param_types(&mut sig.params, &pydoc.params);
501
502        let kind = if is_property {
503            ApiKind::Field
504        } else if in_class {
505            ApiKind::Method
506        } else {
507            ApiKind::Function
508        };
509        let mut it = base(outer, kind, name);
510        it.hidden = is_dunder(&it.name) && !SHOWN_DUNDERS.contains(&it.name.as_str());
511        it.deprecated = deprecation(&attrs).or(pydoc.deprecated);
512        it.doc = pydoc.doc;
513        it.attrs = attrs;
514        if is_property {
515            it.signature = match &sig.returns {
516                Some(r) => format!("{}: {r}", it.name),
517                None => it.name.clone(),
518            };
519        } else {
520            it.signature = signature(def, src);
521            it.sig = Some(sig);
522        }
523        Some(it)
524    }
525
526    fn class(&mut self, def: &Node, outer: &Node, attrs: Vec<String>, _scope: Scope) -> ApiItem {
527        let src = self.src;
528        let name = def
529            .child_by_field_name("name")
530            .map(|n| text(&n, src).to_string())
531            .unwrap_or_default();
532        let bases: Vec<String> = def
533            .child_by_field_name("superclasses")
534            .map(|s| {
535                let mut c = s.walk();
536                s.named_children(&mut c)
537                    .filter(|n| n.kind() != "keyword_argument")
538                    .map(|n| text(&n, src).to_string())
539                    .collect()
540            })
541            .unwrap_or_default();
542        let is_enum = bases.iter().any(|b| {
543            let last = b.rsplit('.').next().unwrap_or(b);
544            last.ends_with("Enum") || last.ends_with("Flag")
545        });
546        let (doc, mut members) = match def.child_by_field_name("body") {
547            Some(b) => self.body(&b, Scope::Class { is_enum }),
548            None => (None, Vec::new()),
549        };
550        let doc = doc.unwrap_or_default();
551        fill_field_docs(&mut members, &doc.attributes);
552
553        let kind = if is_enum {
554            ApiKind::Enum
555        } else {
556            ApiKind::Class
557        };
558        let mut it = base(outer, kind, name);
559        it.signature = signature(def, src);
560        if let Some(tp) = def.child_by_field_name("type_parameters") {
561            it.sig = Some(SigParts {
562                generics: Some(collapse_ws(text(&tp, src))),
563                ..SigParts::default()
564            });
565        }
566        it.deprecated = deprecation(&attrs).or(doc.deprecated);
567        it.doc = doc.doc;
568        it.attrs = attrs;
569        it.members = members;
570        it
571    }
572
573    fn assignment(&mut self, node: &Node, stmt: &Node, scope: Scope) -> Option<ApiItem> {
574        let src = self.src;
575        let left = node.child_by_field_name("left")?;
576        if left.kind() != "identifier" {
577            return None;
578        }
579        let name = text(&left, src).to_string();
580        let ty = node
581            .child_by_field_name("type")
582            .map(|t| collapse_ws(text(&t, src)));
583        let right = node.child_by_field_name("right");
584        if name == "__all__" && scope == Scope::Module {
585            if let Some(r) = right {
586                self.add_exports(&r, true);
587            }
588            return None;
589        }
590        if is_dunder(&name) {
591            return None;
592        }
593        let rhs_call = right
594            .filter(|r| r.kind() == "call")
595            .and_then(|r| r.child_by_field_name("function"))
596            .map(|f| {
597                let t = text(&f, src);
598                t.rsplit('.').next().unwrap_or(t).to_string()
599            });
600        let value = right.map(|r| short_value(text(&r, src)));
601        let sig = match (&ty, &value) {
602            (Some(t), Some(v)) => format!("{name}: {t} = {v}"),
603            (Some(t), None) => format!("{name}: {t}"),
604            (None, Some(v)) => format!("{name} = {v}"),
605            (None, None) => name.clone(),
606        };
607        let is_alias_annotation = ty
608            .as_deref()
609            .is_some_and(|t| t == "TypeAlias" || t.ends_with(".TypeAlias"));
610        let kind = match scope {
611            _ if matches!(
612                rhs_call.as_deref(),
613                Some("TypeVar" | "ParamSpec" | "TypeVarTuple")
614            ) =>
615            {
616                return None;
617            }
618            _ if is_alias_annotation || rhs_call.as_deref() == Some("NewType") => {
619                ApiKind::TypeAlias
620            }
621            Scope::Class { is_enum: true } if ty.is_none() => {
622                if name.starts_with('_') {
623                    return None;
624                }
625                ApiKind::Variant
626            }
627            Scope::Class { .. } if ty.is_some() || UPPER_RE.is_match(&name) => ApiKind::Field,
628            Scope::Class { .. } => return None,
629            Scope::Module if ty.is_some() || UPPER_RE.is_match(&name) => ApiKind::Const,
630            Scope::Module
631                if CAPWORDS_RE.is_match(&name)
632                    && right.is_some_and(|r| r.kind() == "subscript") =>
633            {
634                ApiKind::TypeAlias
635            }
636            Scope::Module => return None,
637        };
638        let mut it = base(stmt, kind, name);
639        it.signature = sig;
640        if kind == ApiKind::Variant {
641            it.visibility = Visibility::Inherited;
642        }
643        Some(it)
644    }
645}
646
647fn short_signature(s: &str) -> String {
648    let s = collapse_ws(s);
649    if s.chars().count() > MAX_SIGNATURE {
650        s.chars().take(MAX_SIGNATURE).collect::<String>() + " …"
651    } else {
652        s
653    }
654}
655
656fn base(node: &Node, kind: ApiKind, name: String) -> ApiItem {
657    ApiItem {
658        visibility: visibility(&name),
659        name,
660        kind,
661        signature: String::new(),
662        sig: None,
663        doc: None,
664        attrs: Vec::new(),
665        deprecated: None,
666        hidden: false,
667        test_only: false,
668        start_line: line(node),
669        end_line: end_line(node),
670        trait_impl: None,
671        self_type: None,
672        members: Vec::new(),
673    }
674}
675
676/// Parameters as written; `*args` and `**kwargs` keep their stars, and the bare `*` and
677/// `/` separators are left out.
678fn params(list: &Node, src: &str) -> Vec<Param> {
679    let mut out = Vec::new();
680    let mut c = list.walk();
681    for p in list.named_children(&mut c) {
682        let field = |n: &str| p.child_by_field_name(n).map(|x| collapse_ws(text(&x, src)));
683        let (name, ty) = match p.kind() {
684            "identifier" => (text(&p, src).to_string(), None),
685            "typed_parameter" => {
686                let mut pc = p.walk();
687                let name = p
688                    .named_children(&mut pc)
689                    .find(|n| n.kind() != "type")
690                    .map(|n| text(&n, src).to_string())
691                    .unwrap_or_default();
692                (name, field("type"))
693            }
694            "default_parameter" | "typed_default_parameter" => {
695                (field("name").unwrap_or_default(), field("type"))
696            }
697            "list_splat_pattern" | "dictionary_splat_pattern" => (text(&p, src).to_string(), None),
698            _ => continue,
699        };
700        out.push(Param {
701            name,
702            ty: ty.unwrap_or_default(),
703        });
704    }
705    out
706}
707
708/// Types from the docstring for parameters without an annotation.
709fn fill_param_types(params: &mut [Param], documented: &[Entry]) {
710    for p in params.iter_mut().filter(|p| p.ty.is_empty()) {
711        let bare = p.name.trim_start_matches('*');
712        let found = documented.iter().find(|e| {
713            e.name
714                .split(',')
715                .any(|n| n.trim().trim_start_matches('*') == bare)
716        });
717        if let Some(ty) = found.and_then(|e| e.ty.as_deref()) {
718            p.ty = ty.to_string();
719        }
720    }
721}
722
723/// Docs from the class docstring's `Attributes` for fields without their own.
724fn fill_field_docs(members: &mut [ApiItem], attributes: &[Entry]) {
725    for m in members
726        .iter_mut()
727        .filter(|m| matches!(m.kind, ApiKind::Field | ApiKind::Variant) && m.doc.is_none())
728    {
729        if let Some(e) = attributes.iter().find(|e| e.name == m.name)
730            && !e.desc.is_empty()
731        {
732            m.doc = pydoc::from_text(&e.desc, m.start_line, m.start_line).doc;
733        }
734    }
735}
736
737/// Drop `@overload` stubs of a name that also has an implementation.
738fn drop_overloads(items: &mut Vec<ApiItem>) {
739    let is_overload = |it: &ApiItem| has_decorator(&it.attrs, &["overload"]);
740    let implemented: Vec<String> = items
741        .iter()
742        .filter(|it| !is_overload(it))
743        .map(|it| it.name.clone())
744        .collect();
745    let mut seen: Vec<String> = Vec::new();
746    items.retain(|it| {
747        if !is_overload(it) {
748            return true;
749        }
750        // Stubs only (a `.pyi`-style module): keep the first.
751        let keep = !implemented.contains(&it.name) && !seen.contains(&it.name);
752        seen.push(it.name.clone());
753        keep
754    });
755}
756
757#[cfg(test)]
758mod tests {
759    use super::*;
760
761    const SAMPLE: &str = r#"#!/usr/bin/env python
762"""Storage engines.
763
764See :class:`Engine`.
765"""
766from __future__ import annotations
767
768import os
769from typing import TypeAlias, TypeVar, overload
770from ._core import Engine, helper as public_helper
771from .. import sibling
772from pkg.sub import *
773
774__all__ = ["Engine", "connect", "MAX_SIZE"]
775__all__ += ["Color"]
776__version__ = "1.0"
777
778T = TypeVar("T")
779
780#: Largest payload, in bytes.
781MAX_SIZE = 1024
782timeout: float = 2.5
783"""Default timeout."""
784logger = object()
785Pair: TypeAlias = tuple[int, int]
786Rows = list[tuple[str, int]]
787type Grid = list[list[int]]
788
789
790async def connect(url: str, *, retries: int = 3, **opts) -> "Engine":
791    """Open a connection.
792
793    Args:
794        url: Where to connect.
795        opts (dict): Driver options.
796    """
797    return Engine(url)
798
799
800def _private():
801    pass
802
803
804@deprecated("use connect")
805def open_db(path):
806    """Old entry point.
807
808    :param path: File path.
809    :type path: str
810    """
811
812
813@dataclass(frozen=True)
814class Config(Base, metaclass=Meta):
815    """Settings.
816
817    Attributes:
818        name: Display name.
819    """
820
821    name: str
822    port: int = 8080
823    """Listening port."""
824    DEFAULT = "x"
825    _secret: str = ""
826    plain = 3
827
828    def __init__(self, name: str) -> None:
829        """Make one."""
830
831    def __repr__(self):
832        return ""
833
834    def __enter__(self):
835        return self
836
837    @property
838    def url(self) -> str:
839        """The URL."""
840        return ""
841
842    @url.setter
843    def url(self, v):
844        pass
845
846    @classmethod
847    def load(cls, path: str) -> "Config":
848        ...
849
850    @staticmethod
851    def check(x):
852        ...
853
854    @overload
855    def get(self, k: int) -> int: ...
856    @overload
857    def get(self, k: str) -> str: ...
858    def get(self, k):
859        """Get one."""
860
861    class Inner:
862        pass
863
864
865class Color(Enum):
866    """Colors."""
867
868    RED = 1
869    """Warm."""
870    GREEN = 2
871
872    def describe(self) -> str:
873        return ""
874
875
876if TYPE_CHECKING:
877    def typed_only() -> None: ...
878
879if __name__ == "__main__":
880    def main(): ...
881
882try:
883    from fast import speedup
884except ImportError:
885    def speedup(): ...
886"#;
887
888    fn find<'a>(items: &'a [ApiItem], name: &str) -> &'a ApiItem {
889        items.iter().find(|i| i.name == name).unwrap_or_else(|| {
890            panic!(
891                "{name} not found in {:?}",
892                items.iter().map(|i| &i.name).collect::<Vec<_>>()
893            )
894        })
895    }
896
897    #[test]
898    fn module_doc_exports_and_reexports() {
899        let f = extract(SAMPLE).unwrap();
900        let d = f.module_doc.as_ref().unwrap();
901        assert_eq!(d.summary, "Storage engines.");
902        assert_eq!(d.links, vec!["Engine"]);
903        assert_eq!(
904            f.exports.as_deref(),
905            Some(&["Engine", "connect", "MAX_SIZE", "Color"].map(String::from)[..])
906        );
907        let re: Vec<(&str, &str)> = f
908            .reexports
909            .iter()
910            .map(|r| (r.path.as_str(), r.name.as_str()))
911            .collect();
912        assert_eq!(
913            re,
914            vec![
915                ("typing.TypeAlias", "TypeAlias"),
916                ("typing.TypeVar", "TypeVar"),
917                ("typing.overload", "overload"),
918                ("._core.Engine", "Engine"),
919                ("._core.helper", "public_helper"),
920                ("..sibling", "sibling"),
921                ("pkg.sub.*", "*"),
922                ("fast.speedup", "speedup"),
923            ]
924        );
925    }
926
927    #[test]
928    fn functions_signatures_and_docs() {
929        let f = extract(SAMPLE).unwrap();
930        let c = find(&f.items, "connect");
931        assert_eq!(c.kind, ApiKind::Function);
932        assert_eq!(c.visibility, Visibility::Public);
933        assert_eq!(
934            c.signature,
935            "async def connect(url: str, *, retries: int = 3, **opts) -> \"Engine\""
936        );
937        let sig = c.sig.as_ref().unwrap();
938        assert!(sig.is_async);
939        let ps: Vec<(&str, &str)> = sig
940            .params
941            .iter()
942            .map(|p| (p.name.as_str(), p.ty.as_str()))
943            .collect();
944        assert_eq!(
945            ps,
946            vec![("url", "str"), ("retries", "int"), ("**opts", "dict")],
947            "unannotated **opts takes its type from the docstring"
948        );
949        assert_eq!(sig.returns.as_deref(), Some("\"Engine\""));
950        assert!(
951            c.doc
952                .as_ref()
953                .unwrap()
954                .markdown
955                .contains("- `url` — Where to connect.")
956        );
957        assert_eq!(find(&f.items, "_private").visibility, Visibility::Private);
958
959        let old = find(&f.items, "open_db");
960        assert_eq!(old.attrs, vec!["deprecated(\"use connect\")"]);
961        assert_eq!(
962            old.deprecated.as_ref().unwrap().note.as_deref(),
963            Some("use connect")
964        );
965        assert_eq!(old.sig.as_ref().unwrap().params[0].ty, "str");
966    }
967
968    #[test]
969    fn constants_and_aliases() {
970        let f = extract(SAMPLE).unwrap();
971        let names: Vec<(&str, ApiKind)> = f
972            .items
973            .iter()
974            .filter(|i| matches!(i.kind, ApiKind::Const | ApiKind::TypeAlias))
975            .map(|i| (i.name.as_str(), i.kind))
976            .collect();
977        assert_eq!(
978            names,
979            vec![
980                ("MAX_SIZE", ApiKind::Const),
981                ("timeout", ApiKind::Const),
982                ("Pair", ApiKind::TypeAlias),
983                ("Rows", ApiKind::TypeAlias),
984                ("Grid", ApiKind::TypeAlias),
985            ],
986            "TypeVars, dunders and plain variables are not API"
987        );
988        let max = find(&f.items, "MAX_SIZE");
989        assert_eq!(max.signature, "MAX_SIZE = 1024");
990        assert_eq!(
991            max.doc.as_ref().unwrap().summary,
992            "Largest payload, in bytes."
993        );
994        let t = find(&f.items, "timeout");
995        assert_eq!(t.signature, "timeout: float = 2.5");
996        assert_eq!(t.doc.as_ref().unwrap().summary, "Default timeout.");
997        assert_eq!(
998            find(&f.items, "Grid").signature,
999            "type Grid = list[list[int]]"
1000        );
1001    }
1002
1003    #[test]
1004    fn classes_dataclasses_properties_and_methods() {
1005        let f = extract(SAMPLE).unwrap();
1006        let c = find(&f.items, "Config");
1007        assert_eq!(c.kind, ApiKind::Class);
1008        assert_eq!(c.signature, "class Config(Base, metaclass=Meta)");
1009        assert_eq!(c.attrs, vec!["dataclass(frozen=True)"]);
1010        assert_eq!(c.start_line, 53, "the item starts at its first decorator");
1011        let names: Vec<&str> = c.members.iter().map(|m| m.name.as_str()).collect();
1012        assert_eq!(
1013            names,
1014            vec![
1015                "name",
1016                "port",
1017                "DEFAULT",
1018                "_secret",
1019                "__init__",
1020                "__repr__",
1021                "__enter__",
1022                "url",
1023                "load",
1024                "check",
1025                "get",
1026                "Inner"
1027            ],
1028            "unannotated lowercase attributes, setters and overload stubs are left out"
1029        );
1030        let name = find(&c.members, "name");
1031        assert_eq!(name.kind, ApiKind::Field);
1032        assert_eq!(name.signature, "name: str");
1033        assert_eq!(
1034            name.doc.as_ref().unwrap().summary,
1035            "Display name.",
1036            "from the class docstring's Attributes"
1037        );
1038        let port = find(&c.members, "port");
1039        assert_eq!(port.signature, "port: int = 8080");
1040        assert_eq!(port.doc.as_ref().unwrap().summary, "Listening port.");
1041        assert_eq!(find(&c.members, "_secret").visibility, Visibility::Private);
1042
1043        let init = find(&c.members, "__init__");
1044        assert_eq!(init.kind, ApiKind::Method);
1045        assert_eq!(init.visibility, Visibility::Inherited);
1046        assert!(!init.hidden, "the constructor is documented");
1047        let isig = init.sig.as_ref().unwrap();
1048        assert_eq!(isig.receiver.as_deref(), Some("self"));
1049        assert_eq!(isig.params.len(), 1);
1050        assert!(find(&c.members, "__repr__").hidden);
1051        assert!(!find(&c.members, "__enter__").hidden);
1052
1053        let url = find(&c.members, "url");
1054        assert_eq!(url.kind, ApiKind::Field);
1055        assert_eq!(url.signature, "url: str");
1056        assert_eq!(url.attrs, vec!["property"]);
1057        assert_eq!(url.doc.as_ref().unwrap().summary, "The URL.");
1058
1059        let load = find(&c.members, "load");
1060        assert_eq!(load.attrs, vec!["classmethod"]);
1061        assert_eq!(load.sig.as_ref().unwrap().receiver.as_deref(), Some("cls"));
1062        let check = find(&c.members, "check");
1063        assert_eq!(check.sig.as_ref().unwrap().receiver, None);
1064        assert_eq!(check.sig.as_ref().unwrap().params[0].name, "x");
1065        assert_eq!(
1066            find(&c.members, "get").doc.as_ref().unwrap().summary,
1067            "Get one."
1068        );
1069        assert_eq!(find(&c.members, "Inner").kind, ApiKind::Class);
1070    }
1071
1072    #[test]
1073    fn enums_and_conditional_blocks() {
1074        let f = extract(SAMPLE).unwrap();
1075        let color = find(&f.items, "Color");
1076        assert_eq!(color.kind, ApiKind::Enum);
1077        let red = find(&color.members, "RED");
1078        assert_eq!(red.kind, ApiKind::Variant);
1079        assert_eq!(red.signature, "RED = 1");
1080        assert_eq!(red.doc.as_ref().unwrap().summary, "Warm.");
1081        assert_eq!(find(&color.members, "describe").kind, ApiKind::Method);
1082
1083        assert!(f.items.iter().any(|i| i.name == "typed_only"));
1084        assert!(!f.items.iter().any(|i| i.name == "main"));
1085        assert!(
1086            !f.items.iter().any(|i| i.name == "speedup"),
1087            "only the try body is walked; the fallback def is in `except`"
1088        );
1089    }
1090
1091    #[test]
1092    fn numpy_docstring_and_deprecated_directive() {
1093        let src = r#"
1094def add(x, y=0):
1095    """Add.
1096
1097    Parameters
1098    ----------
1099    x : int
1100        First.
1101    y : int, optional
1102        Second.
1103
1104    .. deprecated:: 2.0
1105       Use ``plus``.
1106    """
1107"#;
1108        let f = extract(src).unwrap();
1109        let add = find(&f.items, "add");
1110        let ps = &add.sig.as_ref().unwrap().params;
1111        assert_eq!(ps[0].ty, "int");
1112        assert_eq!(ps[1].ty, "int, optional");
1113        let dep = add.deprecated.as_ref().unwrap();
1114        assert_eq!(dep.since.as_deref(), Some("2.0"));
1115        assert_eq!(dep.note.as_deref(), Some("Use `plus`."));
1116    }
1117
1118    #[test]
1119    fn computed_all_is_unknown() {
1120        let f = extract("__all__ = base.__all__ + [\"a\"]\n").unwrap();
1121        assert_eq!(f.exports, None, "partly computed: the name rule applies");
1122        let f = extract("__all__ = [\n    # public\n    \"a\",\n]\n__all__.append('b')\n").unwrap();
1123        assert_eq!(f.exports, Some(vec!["a".into(), "b".into()]));
1124        let f = extract("__all__ = ['a']\n__all__ += helpers.__all__\n").unwrap();
1125        assert_eq!(f.exports, None);
1126    }
1127
1128    #[test]
1129    fn broken_source_still_extracts() {
1130        let f = extract("def ok():\n    '''Fine.'''\n\ndef broken(:\n").unwrap();
1131        assert!(f.items.iter().any(|i| i.name == "ok"));
1132    }
1133
1134    /// Extract every `.py` under `$PULSE_PY_REPO` (run with `--ignored --nocapture`).
1135    #[test]
1136    #[ignore]
1137    fn extract_python_repository() {
1138        let Ok(root) = std::env::var("PULSE_PY_REPO") else {
1139            return;
1140        };
1141        fn walk(dir: &std::path::Path, out: &mut Vec<std::path::PathBuf>) {
1142            for e in std::fs::read_dir(dir).into_iter().flatten().flatten() {
1143                let p = e.path();
1144                if p.is_dir() {
1145                    if !p
1146                        .file_name()
1147                        .is_some_and(|n| n.to_string_lossy().starts_with('.'))
1148                    {
1149                        walk(&p, out);
1150                    }
1151                } else if p.extension().is_some_and(|x| x == "py") {
1152                    out.push(p);
1153                }
1154            }
1155        }
1156        let mut files = Vec::new();
1157        walk(std::path::Path::new(&root), &mut files);
1158        let sources: Vec<String> = files
1159            .iter()
1160            .filter_map(|f| std::fs::read_to_string(f).ok())
1161            .collect();
1162        let parse_start = std::time::Instant::now();
1163        let mut parser = tree_sitter::Parser::new();
1164        parser
1165            .set_language(&tree_sitter_python::LANGUAGE.into())
1166            .unwrap();
1167        for src in &sources {
1168            parser.parse(src, None);
1169        }
1170        println!("parse only: {:?}", parse_start.elapsed());
1171        let start = std::time::Instant::now();
1172        let (mut items, mut documented, mut public, mut examples, mut failed) = (0, 0, 0, 0, 0);
1173        fn count(items: &[ApiItem], n: &mut usize, d: &mut usize, p: &mut usize, e: &mut usize) {
1174            for i in items {
1175                *n += 1;
1176                if let Some(doc) = &i.doc {
1177                    *d += 1;
1178                    *e += doc.examples.len();
1179                }
1180                if i.visibility == Visibility::Public {
1181                    *p += 1;
1182                }
1183                count(&i.members, n, d, p, e);
1184            }
1185        }
1186        for src in &sources {
1187            match extract(src) {
1188                Ok(api) => count(
1189                    &api.items,
1190                    &mut items,
1191                    &mut documented,
1192                    &mut public,
1193                    &mut examples,
1194                ),
1195                Err(_) => failed += 1,
1196            }
1197        }
1198        println!(
1199            "{} files ({failed} failed), {items} items ({documented} documented, {public} public, {examples} examples) in {:?}",
1200            sources.len(),
1201            start.elapsed()
1202        );
1203    }
1204}