1use super::collapse_ws;
32use super::pydoc::{self, Entry, PyDoc};
33use super::{ApiFile, ApiItem, ApiKind, Deprecation, Param, ReExport, SigParts, Visibility};
34use crate::models::Language;
35use anyhow::{Context, Result};
36use regex::Regex;
37use std::sync::LazyLock;
38use tree_sitter::Node;
39
40const MAX_SIGNATURE: usize = 600;
42const MAX_VALUE: usize = 60;
44
45const SHOWN_DUNDERS: &[&str] = &[
47 "__init__",
48 "__call__",
49 "__enter__",
50 "__exit__",
51 "__aenter__",
52 "__aexit__",
53 "__iter__",
54 "__aiter__",
55 "__next__",
56 "__anext__",
57 "__await__",
58 "__getitem__",
59 "__setitem__",
60 "__delitem__",
61 "__contains__",
62 "__len__",
63];
64
65pub fn extract(source: &str) -> Result<ApiFile> {
66 let mut parser = tree_sitter::Parser::new();
67 parser
68 .set_language(&crate::parsers::ParserFactory::get_language_grammar(
69 Language::Python,
70 )?)
71 .context("loading the Python grammar")?;
72 let tree = parser.parse(source, None).context("parsing Python")?;
73 let root = tree.root_node();
74
75 let mut w = Walker {
76 src: source,
77 reexports: Vec::new(),
78 exports: None,
79 exports_dynamic: false,
80 };
81 let (doc, items) = w.body(&root, Scope::Module);
82 Ok(ApiFile {
83 module_doc: doc.and_then(|d| d.doc),
84 items,
85 mod_decls: Vec::new(),
86 reexports: w.reexports,
87 exports: w.exports.filter(|_| !w.exports_dynamic),
90 package: None,
91 })
92}
93
94#[derive(Clone, Copy, PartialEq)]
95enum Scope {
96 Module,
97 Class { is_enum: bool },
98}
99
100struct Walker<'a> {
101 src: &'a str,
102 reexports: Vec<ReExport>,
103 exports: Option<Vec<String>>,
104 exports_dynamic: bool,
106}
107
108fn text<'s>(node: &Node, src: &'s str) -> &'s str {
109 node.utf8_text(src.as_bytes()).unwrap_or("")
110}
111
112fn line(node: &Node) -> u32 {
113 node.start_position().row as u32 + 1
114}
115
116fn end_line(node: &Node) -> u32 {
117 node.end_position().row as u32 + 1
118}
119
120fn is_dunder(name: &str) -> bool {
121 name.len() > 4 && name.starts_with("__") && name.ends_with("__")
122}
123
124fn visibility(name: &str) -> Visibility {
125 if is_dunder(name) {
126 Visibility::Inherited
127 } else if name.starts_with('_') {
128 Visibility::Private
129 } else {
130 Visibility::Public
131 }
132}
133
134static UPPER_RE: LazyLock<Regex> =
135 LazyLock::new(|| Regex::new(r"^_*[A-Z][A-Z0-9_]*$").expect("valid regex"));
136static CAPWORDS_RE: LazyLock<Regex> =
137 LazyLock::new(|| Regex::new(r"^_*[A-Z][A-Za-z0-9]*[a-z][A-Za-z0-9]*$").expect("valid regex"));
138
139fn decorator_name(d: &str) -> &str {
141 let callee = d.split('(').next().unwrap_or(d).trim();
142 callee.rsplit('.').next().unwrap_or(callee)
143}
144
145fn has_decorator(attrs: &[String], names: &[&str]) -> bool {
146 attrs.iter().any(|a| names.contains(&decorator_name(a)))
147}
148
149static DEP_POSITIONAL_RE: LazyLock<Regex> = LazyLock::new(|| {
150 Regex::new(r#"^[\w.]+\(\s*[rRuU]?(?:"([^"]*)"|'([^']*)')"#).expect("valid regex")
151});
152static DEP_SINCE_RE: LazyLock<Regex> = LazyLock::new(|| {
153 Regex::new(r#"\b(?:version|since|deprecated_in)\s*=\s*["']([^"']*)["']"#).expect("valid regex")
154});
155static DEP_NOTE_RE: LazyLock<Regex> = LazyLock::new(|| {
156 Regex::new(r#"\b(?:reason|details|message|msg)\s*=\s*["']([^"']*)["']"#).expect("valid regex")
157});
158
159fn deprecation(attrs: &[String]) -> Option<Deprecation> {
161 let d = attrs.iter().find(|a| decorator_name(a) == "deprecated")?;
162 let cap = |re: &Regex| {
163 re.captures(d)
164 .and_then(|c| c.get(1).or_else(|| c.get(2)))
165 .map(|m| m.as_str().to_string())
166 };
167 Some(Deprecation {
168 since: cap(&DEP_SINCE_RE),
169 note: cap(&DEP_POSITIONAL_RE).or_else(|| cap(&DEP_NOTE_RE)),
170 })
171}
172
173fn short_value(v: &str) -> String {
175 let v = collapse_ws(v);
176 if v.chars().count() > MAX_VALUE {
177 v.chars().take(MAX_VALUE - 1).collect::<String>() + "…"
178 } else {
179 v
180 }
181}
182
183fn signature(node: &Node, src: &str) -> String {
185 let end = node
186 .child_by_field_name("body")
187 .map(|b| b.start_byte())
188 .unwrap_or(node.end_byte());
189 let raw = &src[node.start_byte()..end];
190 let raw = raw.trim_end();
191 let raw = raw.strip_suffix(':').unwrap_or(raw);
192 let mut s = collapse_ws(raw);
193 if s.chars().count() > MAX_SIGNATURE {
194 s = s.chars().take(MAX_SIGNATURE).collect::<String>() + " …";
195 }
196 s
197}
198
199fn docstring(block: &Node, src: &str) -> Option<PyDoc> {
201 let mut c = block.walk();
202 let first = block
203 .named_children(&mut c)
204 .find(|n| n.kind() != "comment")?;
205 string_statement(&first, src)
206}
207
208fn string_statement(stmt: &Node, src: &str) -> Option<PyDoc> {
210 if stmt.kind() != "expression_statement" || stmt.named_child_count() != 1 {
211 return None;
212 }
213 let s = stmt.named_child(0)?;
214 (s.kind() == "string").then(|| pydoc::parse(text(&s, src), line(&s), end_line(&s)))
215}
216
217fn string_list(node: &Node, src: &str, out: &mut Vec<String>) -> bool {
220 match node.kind() {
221 "string" => match pydoc::literal_value(text(node, src)) {
222 Some(v) => {
223 out.push(v);
224 true
225 }
226 None => false,
227 },
228 "list" | "tuple" | "binary_operator" | "parenthesized_expression" | "argument_list" => {
229 let mut c = node.walk();
230 let mut complete = true;
231 for ch in node.named_children(&mut c) {
232 complete &= string_list(&ch, src, out);
233 }
234 complete
235 }
236 "comment" => true,
237 _ => false,
238 }
239}
240
241impl Walker<'_> {
242 fn add_exports(&mut self, value: &Node, replace: bool) {
244 let mut names = Vec::new();
245 let complete = string_list(value, self.src, &mut names);
246 if replace {
247 self.exports = Some(names);
248 self.exports_dynamic = !complete;
249 } else {
250 self.exports.get_or_insert_default().extend(names);
251 self.exports_dynamic |= !complete;
252 }
253 }
254
255 fn body(&mut self, block: &Node, scope: Scope) -> (Option<PyDoc>, Vec<ApiItem>) {
257 let doc = docstring(block, self.src);
258 let mut out = Vec::new();
259 self.statements(block, scope, doc.is_some(), &mut out);
260 drop_overloads(&mut out);
261 (doc, out)
262 }
263
264 fn statements(&mut self, block: &Node, scope: Scope, skip_first: bool, out: &mut Vec<ApiItem>) {
265 let src = self.src;
266 let mut c = block.walk();
267 let children: Vec<Node> = block.named_children(&mut c).collect();
268 let mut skipped = !skip_first;
269 let mut comment_doc: Vec<(String, u32)> = Vec::new();
271 let mut last_assign: Option<usize> = None;
273 for child in &children {
274 if child.kind() == "comment" {
275 match text(child, src).strip_prefix("#:") {
276 Some(t) => comment_doc
277 .push((t.strip_prefix(' ').unwrap_or(t).to_string(), line(child))),
278 None => comment_doc.clear(),
279 }
280 continue;
281 }
282 if !skipped {
283 skipped = true;
284 continue;
285 }
286 let assign = last_assign.take();
287 let comments = std::mem::take(&mut comment_doc);
288 match child.kind() {
289 "function_definition" => {
290 if let Some(it) = self.function(child, child, Vec::new(), scope) {
291 out.push(it);
292 }
293 }
294 "class_definition" => out.push(self.class(child, child, Vec::new(), scope)),
295 "decorated_definition" => {
296 let mut dc = child.walk();
297 let attrs: Vec<String> = child
298 .named_children(&mut dc)
299 .filter(|n| n.kind() == "decorator")
300 .map(|n| collapse_ws(text(&n, src).trim_start_matches('@').trim()))
301 .collect();
302 match child.child_by_field_name("definition") {
303 Some(d) if d.kind() == "function_definition" => {
304 if let Some(it) = self.function(&d, child, attrs, scope) {
305 out.push(it);
306 }
307 }
308 Some(d) if d.kind() == "class_definition" => {
309 out.push(self.class(&d, child, attrs, scope));
310 }
311 _ => {}
312 }
313 }
314 "expression_statement" => {
315 let Some(inner) = child.named_child(0) else {
316 continue;
317 };
318 match inner.kind() {
319 "assignment" => {
320 if let Some(mut it) = self.assignment(&inner, child, scope) {
321 if !comments.is_empty() {
322 let lines: Vec<&str> =
323 comments.iter().map(|(t, _)| t.as_str()).collect();
324 let d = pydoc::from_text(
325 &lines.join("\n"),
326 comments[0].1,
327 comments[comments.len() - 1].1,
328 );
329 it.doc = d.doc;
330 }
331 out.push(it);
332 last_assign = Some(out.len() - 1);
333 }
334 }
335 "string" => {
336 if let Some(i) = assign
338 && out[i].doc.is_none()
339 && let Some(d) = string_statement(child, src)
340 {
341 out[i].doc = d.doc;
342 }
343 }
344 "augmented_assignment" if scope == Scope::Module => {
345 let left = inner.child_by_field_name("left");
346 if left.is_some_and(|l| text(&l, src) == "__all__")
347 && let Some(r) = inner.child_by_field_name("right")
348 {
349 self.add_exports(&r, false);
350 }
351 }
352 "call" if scope == Scope::Module => {
353 let f = inner
354 .child_by_field_name("function")
355 .map(|f| text(&f, src))
356 .unwrap_or("");
357 if (f == "__all__.extend" || f == "__all__.append")
358 && let Some(args) = inner.child_by_field_name("arguments")
359 {
360 self.add_exports(&args, false);
361 }
362 }
363 _ => {}
364 }
365 }
366 "type_alias_statement" => {
367 let name = child
368 .child_by_field_name("left")
369 .map(|l| {
370 let t = text(&l, src);
371 t.split(['[', ' ']).next().unwrap_or(t).to_string()
372 })
373 .unwrap_or_default();
374 let mut it = base(child, ApiKind::TypeAlias, name);
375 it.signature = short_signature(text(child, src));
376 out.push(it);
377 }
378 "import_from_statement" if scope == Scope::Module => self.import_from(child),
379 "if_statement" if scope == Scope::Module => {
380 let cond = child
381 .child_by_field_name("condition")
382 .map(|c| text(&c, src))
383 .unwrap_or("");
384 if !cond.contains("__name__")
385 && let Some(b) = child.child_by_field_name("consequence")
386 {
387 self.statements(&b, scope, false, out);
388 }
389 }
390 "try_statement" if scope == Scope::Module => {
391 if let Some(b) = child.child_by_field_name("body") {
392 self.statements(&b, scope, false, out);
393 }
394 }
395 _ => {}
396 }
397 }
398 }
399
400 fn import_from(&mut self, node: &Node) {
401 let src = self.src;
402 let module = node
403 .child_by_field_name("module_name")
404 .map(|m| collapse_ws(text(&m, src)).replace(' ', ""))
405 .unwrap_or_default();
406 if module == "__future__" {
407 return;
408 }
409 let join = |name: &str| {
410 if module.ends_with('.') {
411 format!("{module}{name}")
412 } else {
413 format!("{module}.{name}")
414 }
415 };
416 let mut c = node.walk();
417 let mut push = |path: String, name: String| {
418 self.reexports.push(ReExport {
419 path,
420 visibility: visibility(&name),
421 name,
422 line: line(node),
423 });
424 };
425 for ch in node.children_by_field_name("name", &mut c) {
426 match ch.kind() {
427 "dotted_name" => {
428 let n = text(&ch, src).to_string();
429 push(join(&n), n);
430 }
431 "aliased_import" => {
432 let n = ch
433 .child_by_field_name("name")
434 .map(|n| text(&n, src).to_string())
435 .unwrap_or_default();
436 let alias = ch
437 .child_by_field_name("alias")
438 .map(|a| text(&a, src).to_string())
439 .unwrap_or_else(|| n.clone());
440 push(join(&n), alias);
441 }
442 _ => {}
443 }
444 }
445 let mut c = node.walk();
446 if node
447 .named_children(&mut c)
448 .any(|n| n.kind() == "wildcard_import")
449 {
450 push(join("*"), "*".into());
451 }
452 }
453
454 fn function(
455 &mut self,
456 def: &Node,
457 outer: &Node,
458 attrs: Vec<String>,
459 scope: Scope,
460 ) -> Option<ApiItem> {
461 let src = self.src;
462 let name = def
463 .child_by_field_name("name")
464 .map(|n| text(&n, src).to_string())
465 .unwrap_or_default();
466 if attrs.iter().any(|a| {
468 a.split('(')
469 .next()
470 .is_some_and(|c| c.ends_with(".setter") || c.ends_with(".deleter"))
471 }) {
472 return None;
473 }
474 let in_class = matches!(scope, Scope::Class { .. });
475 let is_property =
476 in_class && has_decorator(&attrs, &["property", "cached_property", "abstractproperty"]);
477 let is_static = has_decorator(&attrs, &["staticmethod"]);
478 let pydoc = def
479 .child_by_field_name("body")
480 .and_then(|b| docstring(&b, src))
481 .unwrap_or_default();
482
483 let mut sig = SigParts {
484 is_async: def.child(0).is_some_and(|c| c.kind() == "async"),
485 ..SigParts::default()
486 };
487 if let Some(tp) = def.child_by_field_name("type_parameters") {
488 sig.generics = Some(collapse_ws(text(&tp, src)));
489 }
490 if let Some(r) = def.child_by_field_name("return_type") {
491 sig.returns = Some(collapse_ws(text(&r, src)));
492 }
493 if let Some(ps) = def.child_by_field_name("parameters") {
494 sig.params = params(&ps, src);
495 }
496 if in_class && !is_static && !sig.params.is_empty() && !sig.params[0].name.starts_with('*')
497 {
498 sig.receiver = Some(sig.params.remove(0).name);
499 }
500 fill_param_types(&mut sig.params, &pydoc.params);
501
502 let kind = if is_property {
503 ApiKind::Field
504 } else if in_class {
505 ApiKind::Method
506 } else {
507 ApiKind::Function
508 };
509 let mut it = base(outer, kind, name);
510 it.hidden = is_dunder(&it.name) && !SHOWN_DUNDERS.contains(&it.name.as_str());
511 it.deprecated = deprecation(&attrs).or(pydoc.deprecated);
512 it.doc = pydoc.doc;
513 it.attrs = attrs;
514 if is_property {
515 it.signature = match &sig.returns {
516 Some(r) => format!("{}: {r}", it.name),
517 None => it.name.clone(),
518 };
519 } else {
520 it.signature = signature(def, src);
521 it.sig = Some(sig);
522 }
523 Some(it)
524 }
525
526 fn class(&mut self, def: &Node, outer: &Node, attrs: Vec<String>, _scope: Scope) -> ApiItem {
527 let src = self.src;
528 let name = def
529 .child_by_field_name("name")
530 .map(|n| text(&n, src).to_string())
531 .unwrap_or_default();
532 let bases: Vec<String> = def
533 .child_by_field_name("superclasses")
534 .map(|s| {
535 let mut c = s.walk();
536 s.named_children(&mut c)
537 .filter(|n| n.kind() != "keyword_argument")
538 .map(|n| text(&n, src).to_string())
539 .collect()
540 })
541 .unwrap_or_default();
542 let is_enum = bases.iter().any(|b| {
543 let last = b.rsplit('.').next().unwrap_or(b);
544 last.ends_with("Enum") || last.ends_with("Flag")
545 });
546 let (doc, mut members) = match def.child_by_field_name("body") {
547 Some(b) => self.body(&b, Scope::Class { is_enum }),
548 None => (None, Vec::new()),
549 };
550 let doc = doc.unwrap_or_default();
551 fill_field_docs(&mut members, &doc.attributes);
552
553 let kind = if is_enum {
554 ApiKind::Enum
555 } else {
556 ApiKind::Class
557 };
558 let mut it = base(outer, kind, name);
559 it.signature = signature(def, src);
560 if let Some(tp) = def.child_by_field_name("type_parameters") {
561 it.sig = Some(SigParts {
562 generics: Some(collapse_ws(text(&tp, src))),
563 ..SigParts::default()
564 });
565 }
566 it.deprecated = deprecation(&attrs).or(doc.deprecated);
567 it.doc = doc.doc;
568 it.attrs = attrs;
569 it.members = members;
570 it
571 }
572
573 fn assignment(&mut self, node: &Node, stmt: &Node, scope: Scope) -> Option<ApiItem> {
574 let src = self.src;
575 let left = node.child_by_field_name("left")?;
576 if left.kind() != "identifier" {
577 return None;
578 }
579 let name = text(&left, src).to_string();
580 let ty = node
581 .child_by_field_name("type")
582 .map(|t| collapse_ws(text(&t, src)));
583 let right = node.child_by_field_name("right");
584 if name == "__all__" && scope == Scope::Module {
585 if let Some(r) = right {
586 self.add_exports(&r, true);
587 }
588 return None;
589 }
590 if is_dunder(&name) {
591 return None;
592 }
593 let rhs_call = right
594 .filter(|r| r.kind() == "call")
595 .and_then(|r| r.child_by_field_name("function"))
596 .map(|f| {
597 let t = text(&f, src);
598 t.rsplit('.').next().unwrap_or(t).to_string()
599 });
600 let value = right.map(|r| short_value(text(&r, src)));
601 let sig = match (&ty, &value) {
602 (Some(t), Some(v)) => format!("{name}: {t} = {v}"),
603 (Some(t), None) => format!("{name}: {t}"),
604 (None, Some(v)) => format!("{name} = {v}"),
605 (None, None) => name.clone(),
606 };
607 let is_alias_annotation = ty
608 .as_deref()
609 .is_some_and(|t| t == "TypeAlias" || t.ends_with(".TypeAlias"));
610 let kind = match scope {
611 _ if matches!(
612 rhs_call.as_deref(),
613 Some("TypeVar" | "ParamSpec" | "TypeVarTuple")
614 ) =>
615 {
616 return None;
617 }
618 _ if is_alias_annotation || rhs_call.as_deref() == Some("NewType") => {
619 ApiKind::TypeAlias
620 }
621 Scope::Class { is_enum: true } if ty.is_none() => {
622 if name.starts_with('_') {
623 return None;
624 }
625 ApiKind::Variant
626 }
627 Scope::Class { .. } if ty.is_some() || UPPER_RE.is_match(&name) => ApiKind::Field,
628 Scope::Class { .. } => return None,
629 Scope::Module if ty.is_some() || UPPER_RE.is_match(&name) => ApiKind::Const,
630 Scope::Module
631 if CAPWORDS_RE.is_match(&name)
632 && right.is_some_and(|r| r.kind() == "subscript") =>
633 {
634 ApiKind::TypeAlias
635 }
636 Scope::Module => return None,
637 };
638 let mut it = base(stmt, kind, name);
639 it.signature = sig;
640 if kind == ApiKind::Variant {
641 it.visibility = Visibility::Inherited;
642 }
643 Some(it)
644 }
645}
646
647fn short_signature(s: &str) -> String {
648 let s = collapse_ws(s);
649 if s.chars().count() > MAX_SIGNATURE {
650 s.chars().take(MAX_SIGNATURE).collect::<String>() + " …"
651 } else {
652 s
653 }
654}
655
656fn base(node: &Node, kind: ApiKind, name: String) -> ApiItem {
657 ApiItem {
658 visibility: visibility(&name),
659 name,
660 kind,
661 signature: String::new(),
662 sig: None,
663 doc: None,
664 attrs: Vec::new(),
665 deprecated: None,
666 hidden: false,
667 test_only: false,
668 start_line: line(node),
669 end_line: end_line(node),
670 trait_impl: None,
671 self_type: None,
672 members: Vec::new(),
673 }
674}
675
676fn params(list: &Node, src: &str) -> Vec<Param> {
679 let mut out = Vec::new();
680 let mut c = list.walk();
681 for p in list.named_children(&mut c) {
682 let field = |n: &str| p.child_by_field_name(n).map(|x| collapse_ws(text(&x, src)));
683 let (name, ty) = match p.kind() {
684 "identifier" => (text(&p, src).to_string(), None),
685 "typed_parameter" => {
686 let mut pc = p.walk();
687 let name = p
688 .named_children(&mut pc)
689 .find(|n| n.kind() != "type")
690 .map(|n| text(&n, src).to_string())
691 .unwrap_or_default();
692 (name, field("type"))
693 }
694 "default_parameter" | "typed_default_parameter" => {
695 (field("name").unwrap_or_default(), field("type"))
696 }
697 "list_splat_pattern" | "dictionary_splat_pattern" => (text(&p, src).to_string(), None),
698 _ => continue,
699 };
700 out.push(Param {
701 name,
702 ty: ty.unwrap_or_default(),
703 });
704 }
705 out
706}
707
708fn fill_param_types(params: &mut [Param], documented: &[Entry]) {
710 for p in params.iter_mut().filter(|p| p.ty.is_empty()) {
711 let bare = p.name.trim_start_matches('*');
712 let found = documented.iter().find(|e| {
713 e.name
714 .split(',')
715 .any(|n| n.trim().trim_start_matches('*') == bare)
716 });
717 if let Some(ty) = found.and_then(|e| e.ty.as_deref()) {
718 p.ty = ty.to_string();
719 }
720 }
721}
722
723fn fill_field_docs(members: &mut [ApiItem], attributes: &[Entry]) {
725 for m in members
726 .iter_mut()
727 .filter(|m| matches!(m.kind, ApiKind::Field | ApiKind::Variant) && m.doc.is_none())
728 {
729 if let Some(e) = attributes.iter().find(|e| e.name == m.name)
730 && !e.desc.is_empty()
731 {
732 m.doc = pydoc::from_text(&e.desc, m.start_line, m.start_line).doc;
733 }
734 }
735}
736
737fn drop_overloads(items: &mut Vec<ApiItem>) {
739 let is_overload = |it: &ApiItem| has_decorator(&it.attrs, &["overload"]);
740 let implemented: Vec<String> = items
741 .iter()
742 .filter(|it| !is_overload(it))
743 .map(|it| it.name.clone())
744 .collect();
745 let mut seen: Vec<String> = Vec::new();
746 items.retain(|it| {
747 if !is_overload(it) {
748 return true;
749 }
750 let keep = !implemented.contains(&it.name) && !seen.contains(&it.name);
752 seen.push(it.name.clone());
753 keep
754 });
755}
756
757#[cfg(test)]
758mod tests {
759 use super::*;
760
761 const SAMPLE: &str = r#"#!/usr/bin/env python
762"""Storage engines.
763
764See :class:`Engine`.
765"""
766from __future__ import annotations
767
768import os
769from typing import TypeAlias, TypeVar, overload
770from ._core import Engine, helper as public_helper
771from .. import sibling
772from pkg.sub import *
773
774__all__ = ["Engine", "connect", "MAX_SIZE"]
775__all__ += ["Color"]
776__version__ = "1.0"
777
778T = TypeVar("T")
779
780#: Largest payload, in bytes.
781MAX_SIZE = 1024
782timeout: float = 2.5
783"""Default timeout."""
784logger = object()
785Pair: TypeAlias = tuple[int, int]
786Rows = list[tuple[str, int]]
787type Grid = list[list[int]]
788
789
790async def connect(url: str, *, retries: int = 3, **opts) -> "Engine":
791 """Open a connection.
792
793 Args:
794 url: Where to connect.
795 opts (dict): Driver options.
796 """
797 return Engine(url)
798
799
800def _private():
801 pass
802
803
804@deprecated("use connect")
805def open_db(path):
806 """Old entry point.
807
808 :param path: File path.
809 :type path: str
810 """
811
812
813@dataclass(frozen=True)
814class Config(Base, metaclass=Meta):
815 """Settings.
816
817 Attributes:
818 name: Display name.
819 """
820
821 name: str
822 port: int = 8080
823 """Listening port."""
824 DEFAULT = "x"
825 _secret: str = ""
826 plain = 3
827
828 def __init__(self, name: str) -> None:
829 """Make one."""
830
831 def __repr__(self):
832 return ""
833
834 def __enter__(self):
835 return self
836
837 @property
838 def url(self) -> str:
839 """The URL."""
840 return ""
841
842 @url.setter
843 def url(self, v):
844 pass
845
846 @classmethod
847 def load(cls, path: str) -> "Config":
848 ...
849
850 @staticmethod
851 def check(x):
852 ...
853
854 @overload
855 def get(self, k: int) -> int: ...
856 @overload
857 def get(self, k: str) -> str: ...
858 def get(self, k):
859 """Get one."""
860
861 class Inner:
862 pass
863
864
865class Color(Enum):
866 """Colors."""
867
868 RED = 1
869 """Warm."""
870 GREEN = 2
871
872 def describe(self) -> str:
873 return ""
874
875
876if TYPE_CHECKING:
877 def typed_only() -> None: ...
878
879if __name__ == "__main__":
880 def main(): ...
881
882try:
883 from fast import speedup
884except ImportError:
885 def speedup(): ...
886"#;
887
888 fn find<'a>(items: &'a [ApiItem], name: &str) -> &'a ApiItem {
889 items.iter().find(|i| i.name == name).unwrap_or_else(|| {
890 panic!(
891 "{name} not found in {:?}",
892 items.iter().map(|i| &i.name).collect::<Vec<_>>()
893 )
894 })
895 }
896
897 #[test]
898 fn module_doc_exports_and_reexports() {
899 let f = extract(SAMPLE).unwrap();
900 let d = f.module_doc.as_ref().unwrap();
901 assert_eq!(d.summary, "Storage engines.");
902 assert_eq!(d.links, vec!["Engine"]);
903 assert_eq!(
904 f.exports.as_deref(),
905 Some(&["Engine", "connect", "MAX_SIZE", "Color"].map(String::from)[..])
906 );
907 let re: Vec<(&str, &str)> = f
908 .reexports
909 .iter()
910 .map(|r| (r.path.as_str(), r.name.as_str()))
911 .collect();
912 assert_eq!(
913 re,
914 vec![
915 ("typing.TypeAlias", "TypeAlias"),
916 ("typing.TypeVar", "TypeVar"),
917 ("typing.overload", "overload"),
918 ("._core.Engine", "Engine"),
919 ("._core.helper", "public_helper"),
920 ("..sibling", "sibling"),
921 ("pkg.sub.*", "*"),
922 ("fast.speedup", "speedup"),
923 ]
924 );
925 }
926
927 #[test]
928 fn functions_signatures_and_docs() {
929 let f = extract(SAMPLE).unwrap();
930 let c = find(&f.items, "connect");
931 assert_eq!(c.kind, ApiKind::Function);
932 assert_eq!(c.visibility, Visibility::Public);
933 assert_eq!(
934 c.signature,
935 "async def connect(url: str, *, retries: int = 3, **opts) -> \"Engine\""
936 );
937 let sig = c.sig.as_ref().unwrap();
938 assert!(sig.is_async);
939 let ps: Vec<(&str, &str)> = sig
940 .params
941 .iter()
942 .map(|p| (p.name.as_str(), p.ty.as_str()))
943 .collect();
944 assert_eq!(
945 ps,
946 vec![("url", "str"), ("retries", "int"), ("**opts", "dict")],
947 "unannotated **opts takes its type from the docstring"
948 );
949 assert_eq!(sig.returns.as_deref(), Some("\"Engine\""));
950 assert!(
951 c.doc
952 .as_ref()
953 .unwrap()
954 .markdown
955 .contains("- `url` — Where to connect.")
956 );
957 assert_eq!(find(&f.items, "_private").visibility, Visibility::Private);
958
959 let old = find(&f.items, "open_db");
960 assert_eq!(old.attrs, vec!["deprecated(\"use connect\")"]);
961 assert_eq!(
962 old.deprecated.as_ref().unwrap().note.as_deref(),
963 Some("use connect")
964 );
965 assert_eq!(old.sig.as_ref().unwrap().params[0].ty, "str");
966 }
967
968 #[test]
969 fn constants_and_aliases() {
970 let f = extract(SAMPLE).unwrap();
971 let names: Vec<(&str, ApiKind)> = f
972 .items
973 .iter()
974 .filter(|i| matches!(i.kind, ApiKind::Const | ApiKind::TypeAlias))
975 .map(|i| (i.name.as_str(), i.kind))
976 .collect();
977 assert_eq!(
978 names,
979 vec![
980 ("MAX_SIZE", ApiKind::Const),
981 ("timeout", ApiKind::Const),
982 ("Pair", ApiKind::TypeAlias),
983 ("Rows", ApiKind::TypeAlias),
984 ("Grid", ApiKind::TypeAlias),
985 ],
986 "TypeVars, dunders and plain variables are not API"
987 );
988 let max = find(&f.items, "MAX_SIZE");
989 assert_eq!(max.signature, "MAX_SIZE = 1024");
990 assert_eq!(
991 max.doc.as_ref().unwrap().summary,
992 "Largest payload, in bytes."
993 );
994 let t = find(&f.items, "timeout");
995 assert_eq!(t.signature, "timeout: float = 2.5");
996 assert_eq!(t.doc.as_ref().unwrap().summary, "Default timeout.");
997 assert_eq!(
998 find(&f.items, "Grid").signature,
999 "type Grid = list[list[int]]"
1000 );
1001 }
1002
1003 #[test]
1004 fn classes_dataclasses_properties_and_methods() {
1005 let f = extract(SAMPLE).unwrap();
1006 let c = find(&f.items, "Config");
1007 assert_eq!(c.kind, ApiKind::Class);
1008 assert_eq!(c.signature, "class Config(Base, metaclass=Meta)");
1009 assert_eq!(c.attrs, vec!["dataclass(frozen=True)"]);
1010 assert_eq!(c.start_line, 53, "the item starts at its first decorator");
1011 let names: Vec<&str> = c.members.iter().map(|m| m.name.as_str()).collect();
1012 assert_eq!(
1013 names,
1014 vec![
1015 "name",
1016 "port",
1017 "DEFAULT",
1018 "_secret",
1019 "__init__",
1020 "__repr__",
1021 "__enter__",
1022 "url",
1023 "load",
1024 "check",
1025 "get",
1026 "Inner"
1027 ],
1028 "unannotated lowercase attributes, setters and overload stubs are left out"
1029 );
1030 let name = find(&c.members, "name");
1031 assert_eq!(name.kind, ApiKind::Field);
1032 assert_eq!(name.signature, "name: str");
1033 assert_eq!(
1034 name.doc.as_ref().unwrap().summary,
1035 "Display name.",
1036 "from the class docstring's Attributes"
1037 );
1038 let port = find(&c.members, "port");
1039 assert_eq!(port.signature, "port: int = 8080");
1040 assert_eq!(port.doc.as_ref().unwrap().summary, "Listening port.");
1041 assert_eq!(find(&c.members, "_secret").visibility, Visibility::Private);
1042
1043 let init = find(&c.members, "__init__");
1044 assert_eq!(init.kind, ApiKind::Method);
1045 assert_eq!(init.visibility, Visibility::Inherited);
1046 assert!(!init.hidden, "the constructor is documented");
1047 let isig = init.sig.as_ref().unwrap();
1048 assert_eq!(isig.receiver.as_deref(), Some("self"));
1049 assert_eq!(isig.params.len(), 1);
1050 assert!(find(&c.members, "__repr__").hidden);
1051 assert!(!find(&c.members, "__enter__").hidden);
1052
1053 let url = find(&c.members, "url");
1054 assert_eq!(url.kind, ApiKind::Field);
1055 assert_eq!(url.signature, "url: str");
1056 assert_eq!(url.attrs, vec!["property"]);
1057 assert_eq!(url.doc.as_ref().unwrap().summary, "The URL.");
1058
1059 let load = find(&c.members, "load");
1060 assert_eq!(load.attrs, vec!["classmethod"]);
1061 assert_eq!(load.sig.as_ref().unwrap().receiver.as_deref(), Some("cls"));
1062 let check = find(&c.members, "check");
1063 assert_eq!(check.sig.as_ref().unwrap().receiver, None);
1064 assert_eq!(check.sig.as_ref().unwrap().params[0].name, "x");
1065 assert_eq!(
1066 find(&c.members, "get").doc.as_ref().unwrap().summary,
1067 "Get one."
1068 );
1069 assert_eq!(find(&c.members, "Inner").kind, ApiKind::Class);
1070 }
1071
1072 #[test]
1073 fn enums_and_conditional_blocks() {
1074 let f = extract(SAMPLE).unwrap();
1075 let color = find(&f.items, "Color");
1076 assert_eq!(color.kind, ApiKind::Enum);
1077 let red = find(&color.members, "RED");
1078 assert_eq!(red.kind, ApiKind::Variant);
1079 assert_eq!(red.signature, "RED = 1");
1080 assert_eq!(red.doc.as_ref().unwrap().summary, "Warm.");
1081 assert_eq!(find(&color.members, "describe").kind, ApiKind::Method);
1082
1083 assert!(f.items.iter().any(|i| i.name == "typed_only"));
1084 assert!(!f.items.iter().any(|i| i.name == "main"));
1085 assert!(
1086 !f.items.iter().any(|i| i.name == "speedup"),
1087 "only the try body is walked; the fallback def is in `except`"
1088 );
1089 }
1090
1091 #[test]
1092 fn numpy_docstring_and_deprecated_directive() {
1093 let src = r#"
1094def add(x, y=0):
1095 """Add.
1096
1097 Parameters
1098 ----------
1099 x : int
1100 First.
1101 y : int, optional
1102 Second.
1103
1104 .. deprecated:: 2.0
1105 Use ``plus``.
1106 """
1107"#;
1108 let f = extract(src).unwrap();
1109 let add = find(&f.items, "add");
1110 let ps = &add.sig.as_ref().unwrap().params;
1111 assert_eq!(ps[0].ty, "int");
1112 assert_eq!(ps[1].ty, "int, optional");
1113 let dep = add.deprecated.as_ref().unwrap();
1114 assert_eq!(dep.since.as_deref(), Some("2.0"));
1115 assert_eq!(dep.note.as_deref(), Some("Use `plus`."));
1116 }
1117
1118 #[test]
1119 fn computed_all_is_unknown() {
1120 let f = extract("__all__ = base.__all__ + [\"a\"]\n").unwrap();
1121 assert_eq!(f.exports, None, "partly computed: the name rule applies");
1122 let f = extract("__all__ = [\n # public\n \"a\",\n]\n__all__.append('b')\n").unwrap();
1123 assert_eq!(f.exports, Some(vec!["a".into(), "b".into()]));
1124 let f = extract("__all__ = ['a']\n__all__ += helpers.__all__\n").unwrap();
1125 assert_eq!(f.exports, None);
1126 }
1127
1128 #[test]
1129 fn broken_source_still_extracts() {
1130 let f = extract("def ok():\n '''Fine.'''\n\ndef broken(:\n").unwrap();
1131 assert!(f.items.iter().any(|i| i.name == "ok"));
1132 }
1133
1134 #[test]
1136 #[ignore]
1137 fn extract_python_repository() {
1138 let Ok(root) = std::env::var("PULSE_PY_REPO") else {
1139 return;
1140 };
1141 fn walk(dir: &std::path::Path, out: &mut Vec<std::path::PathBuf>) {
1142 for e in std::fs::read_dir(dir).into_iter().flatten().flatten() {
1143 let p = e.path();
1144 if p.is_dir() {
1145 if !p
1146 .file_name()
1147 .is_some_and(|n| n.to_string_lossy().starts_with('.'))
1148 {
1149 walk(&p, out);
1150 }
1151 } else if p.extension().is_some_and(|x| x == "py") {
1152 out.push(p);
1153 }
1154 }
1155 }
1156 let mut files = Vec::new();
1157 walk(std::path::Path::new(&root), &mut files);
1158 let sources: Vec<String> = files
1159 .iter()
1160 .filter_map(|f| std::fs::read_to_string(f).ok())
1161 .collect();
1162 let parse_start = std::time::Instant::now();
1163 let mut parser = tree_sitter::Parser::new();
1164 parser
1165 .set_language(&tree_sitter_python::LANGUAGE.into())
1166 .unwrap();
1167 for src in &sources {
1168 parser.parse(src, None);
1169 }
1170 println!("parse only: {:?}", parse_start.elapsed());
1171 let start = std::time::Instant::now();
1172 let (mut items, mut documented, mut public, mut examples, mut failed) = (0, 0, 0, 0, 0);
1173 fn count(items: &[ApiItem], n: &mut usize, d: &mut usize, p: &mut usize, e: &mut usize) {
1174 for i in items {
1175 *n += 1;
1176 if let Some(doc) = &i.doc {
1177 *d += 1;
1178 *e += doc.examples.len();
1179 }
1180 if i.visibility == Visibility::Public {
1181 *p += 1;
1182 }
1183 count(&i.members, n, d, p, e);
1184 }
1185 }
1186 for src in &sources {
1187 match extract(src) {
1188 Ok(api) => count(
1189 &api.items,
1190 &mut items,
1191 &mut documented,
1192 &mut public,
1193 &mut examples,
1194 ),
1195 Err(_) => failed += 1,
1196 }
1197 }
1198 println!(
1199 "{} files ({failed} failed), {items} items ({documented} documented, {public} public, {examples} examples) in {:?}",
1200 sources.len(),
1201 start.elapsed()
1202 );
1203 }
1204}