1use std::cmp::Ordering;
9use std::collections::{BTreeMap, BTreeSet};
10
11use serde::{Deserialize, Serialize};
12
13use crate::{ExtractionLimitExceeded, ExtractionTracker};
14
15const HTTP_METHODS: [&str; 8] = [
16 "DELETE", "GET", "HEAD", "OPTIONS", "PATCH", "POST", "PUT", "TRACE",
17];
18
19#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
21#[serde(rename_all = "snake_case")]
22pub enum SourceLanguage {
23 JavaScript,
25 TypeScript,
27 Rust,
29 Python,
31 Go,
33 Java,
35}
36
37#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
39#[serde(rename_all = "snake_case")]
40pub enum SourceFramework {
41 Fetch,
43 Axios,
45 Express,
47 Fastify,
49 NestJs,
51 NextJs,
53 Axum,
55 ActixWeb,
57 Utoipa,
59 Reqwest,
61 RustTest,
63 TokioTest,
65 Rstest,
67 FastApi,
69 Flask,
71 Requests,
73 Httpx,
75 AioHttp,
77 PythonHttpRegistry,
79 FactoryBoy,
81 Pytest,
83 Unittest,
85 PythonTest,
87 GoNetHttp,
89 Gin,
91 Chi,
93 SpringMvc,
95 WebClient,
97 Feign,
99}
100
101#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
103#[serde(rename_all = "snake_case")]
104pub enum SourceRole {
105 Provider,
107 Consumer,
109 Test,
111 Factory,
113}
114
115#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
117#[serde(rename_all = "snake_case")]
118pub enum SourceEpistemicStatus {
119 Confirmed,
121 Ambiguous,
123 Incomplete,
125}
126
127#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
129pub struct SourceLineRange {
130 pub start: u32,
132 pub end: u32,
134}
135
136#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
138#[serde(rename_all = "snake_case")]
139pub enum SourceWarning {
140 DynamicMethod,
142 DynamicPath,
144 UnsupportedLiteralPath,
146 MissingSymbol,
148 SyntaxErrorRecovery,
150 AdvisoryDeclaration,
152}
153
154#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
161pub struct SourceObservation {
162 pub language: SourceLanguage,
164 pub framework: SourceFramework,
166 pub role: SourceRole,
168 pub method: Option<String>,
170 pub path: Option<String>,
172 pub symbol_name: Option<String>,
174 pub related_symbol: Option<String>,
176 pub related_path: Option<String>,
178 pub lines: SourceLineRange,
180 pub status: SourceEpistemicStatus,
182 pub confidence: f32,
184 pub warnings: Vec<SourceWarning>,
186}
187
188pub(crate) struct SourceObservationCollector<'a> {
189 observations: Vec<SourceObservation>,
190 tracker: Option<&'a mut ExtractionTracker>,
191 error: Option<ExtractionLimitExceeded>,
192}
193
194impl<'a> SourceObservationCollector<'a> {
195 pub(crate) fn unbounded() -> Self {
196 Self {
197 observations: Vec::new(),
198 tracker: None,
199 error: None,
200 }
201 }
202
203 pub(crate) fn bounded(tracker: &'a mut ExtractionTracker) -> Self {
204 Self {
205 observations: Vec::new(),
206 tracker: Some(tracker),
207 error: None,
208 }
209 }
210
211 pub(crate) fn push(&mut self, observation: SourceObservation) {
212 if self.error.is_some() {
213 return;
214 }
215 if let Some(tracker) = self.tracker.as_deref_mut()
216 && let Err(error) = tracker
217 .charge_observation(1)
218 .and_then(|()| charge_source_observation_values(&observation, tracker))
219 {
220 self.error = Some(error);
221 return;
222 }
223 self.observations.push(observation);
224 }
225
226 pub(crate) fn into_result(self) -> Result<Vec<SourceObservation>, ExtractionLimitExceeded> {
227 match self.error {
228 Some(error) => Err(error),
229 None => Ok(self.observations),
230 }
231 }
232
233 pub(crate) fn into_unbounded(self) -> Vec<SourceObservation> {
234 self.observations
235 }
236}
237
238pub(crate) fn charge_source_observation_values(
239 observation: &SourceObservation,
240 tracker: &mut ExtractionTracker,
241) -> Result<(), ExtractionLimitExceeded> {
242 if let Some(method) = &observation.method {
243 tracker.charge_identifier(method)?;
244 }
245 if let Some(path) = &observation.path {
246 tracker.charge_portable_path(path)?;
247 }
248 for symbol in [
249 observation.symbol_name.as_deref(),
250 observation.related_symbol.as_deref(),
251 ]
252 .into_iter()
253 .flatten()
254 {
255 tracker.charge_identifier(symbol)?;
256 }
257 if let Some(path) = &observation.related_path {
258 tracker.charge_portable_path(path)?;
259 }
260 Ok(())
261}
262
263#[must_use]
269pub fn parse_rust_source(source: &str) -> Vec<SourceObservation> {
270 let collector = collect_rust_source(source, SourceObservationCollector::unbounded());
271 finish(collector.into_unbounded())
272}
273
274fn collect_rust_source<'a>(
275 source: &str,
276 mut observations: SourceObservationCollector<'a>,
277) -> SourceObservationCollector<'a> {
278 let tokens = lex_rust(source);
279 let functions = rust_functions(&tokens);
280 parse_rust_attributes(&tokens, &mut observations);
281 if has_ident(&tokens, "axum") {
282 parse_axum_routes(&tokens, &mut observations);
283 }
284 if has_ident(&tokens, "actix_web") {
285 parse_actix_builder_routes(&tokens, &mut observations);
286 }
287 if has_ident(&tokens, "reqwest") {
288 parse_reqwest_calls(&tokens, &functions, &mut observations);
289 }
290 observations
291}
292
293pub fn parse_rust_source_with_tracker(
300 source: &str,
301 tracker: &mut ExtractionTracker,
302) -> Result<Vec<SourceObservation>, ExtractionLimitExceeded> {
303 let observations = collect_rust_source(source, SourceObservationCollector::bounded(tracker));
304 Ok(finish(observations.into_result()?))
305}
306
307#[must_use]
315pub fn parse_python_source(source: &str) -> Vec<SourceObservation> {
316 let collector = collect_python_source(source, SourceObservationCollector::unbounded());
317 finish(collector.into_unbounded())
318}
319
320fn collect_python_source<'a>(
321 source: &str,
322 mut observations: SourceObservationCollector<'a>,
323) -> SourceObservationCollector<'a> {
324 let tokens = lex_python(source);
325 let functions = python_functions(source, &tokens);
326 let contexts = PythonContexts::discover(&tokens);
327 parse_python_routes(&tokens, &contexts, &mut observations);
328 parse_python_http_registries(&tokens, &mut observations);
329 parse_python_http_calls(&tokens, &functions, &contexts, &mut observations);
330 parse_python_factories(source, &tokens, &mut observations);
331 parse_python_tests(source, &tokens, &mut observations);
332 observations
333}
334
335pub fn parse_python_source_with_tracker(
342 source: &str,
343 tracker: &mut ExtractionTracker,
344) -> Result<Vec<SourceObservation>, ExtractionLimitExceeded> {
345 let observations = collect_python_source(source, SourceObservationCollector::bounded(tracker));
346 Ok(finish(observations.into_result()?))
347}
348
349#[must_use]
353pub fn normalize_source_http_path(path: &str) -> String {
354 let trimmed = path.trim();
355 if trimmed.is_empty() || trimmed == "/" {
356 return "/".to_owned();
357 }
358 let mut normalized = String::with_capacity(trimmed.len() + 1);
359 if !trimmed.starts_with('/') {
360 normalized.push('/');
361 }
362 let mut previous_slash = false;
363 for character in trimmed.chars() {
364 if character == '/' {
365 if !previous_slash {
366 normalized.push('/');
367 }
368 previous_slash = true;
369 } else {
370 normalized.push(character);
371 previous_slash = false;
372 }
373 }
374 while normalized.len() > 1 && normalized.ends_with('/') {
375 normalized.pop();
376 }
377 normalized
378}
379
380#[derive(Debug, Clone, PartialEq, Eq)]
381enum TokenKind {
382 Ident(String),
383 Literal(Option<String>),
384 Punct(char),
385}
386
387#[derive(Debug, Clone, PartialEq, Eq)]
388struct Token {
389 kind: TokenKind,
390 line: u32,
391 end_line: u32,
392}
393
394impl Token {
395 fn is_ident(&self, expected: &str) -> bool {
396 matches!(&self.kind, TokenKind::Ident(value) if value == expected)
397 }
398
399 fn is_punct(&self, expected: char) -> bool {
400 self.kind == TokenKind::Punct(expected)
401 }
402
403 fn literal(&self) -> Option<&str> {
404 match &self.kind {
405 TokenKind::Literal(Some(value)) => Some(value),
406 _ => None,
407 }
408 }
409
410 fn ident(&self) -> Option<&str> {
411 match &self.kind {
412 TokenKind::Ident(value) => Some(value),
413 _ => None,
414 }
415 }
416}
417
418#[derive(Debug, Clone)]
419struct FunctionSpan {
420 name: String,
421 start_token: usize,
422 end_token: usize,
423}
424
425fn has_ident(tokens: &[Token], name: &str) -> bool {
426 tokens.iter().any(|token| token.is_ident(name))
427}
428
429fn lex_rust(source: &str) -> Vec<Token> {
430 let characters = source.chars().collect::<Vec<_>>();
431 let mut tokens = Vec::new();
432 let mut index = 0;
433 let mut line = 1_u32;
434 while index < characters.len() {
435 match characters[index] {
436 '\n' => {
437 line += 1;
438 index += 1;
439 }
440 character if character.is_whitespace() => index += 1,
441 '/' if characters.get(index + 1) == Some(&'/') => {
442 index += 2;
443 while index < characters.len() && characters[index] != '\n' {
444 index += 1;
445 }
446 }
447 '/' if characters.get(index + 1) == Some(&'*') => {
448 index += 2;
449 let mut depth = 1_u32;
450 while index < characters.len() && depth > 0 {
451 if characters[index] == '\n' {
452 line += 1;
453 index += 1;
454 } else if characters[index] == '/' && characters.get(index + 1) == Some(&'*') {
455 depth += 1;
456 index += 2;
457 } else if characters[index] == '*' && characters.get(index + 1) == Some(&'/') {
458 depth -= 1;
459 index += 2;
460 } else {
461 index += 1;
462 }
463 }
464 }
465 'r' if rust_raw_string_start(&characters, index).is_some() => {
466 let start_line = line;
467 let (next, value) = consume_rust_raw_string(&characters, index, &mut line);
468 tokens.push(Token {
469 kind: TokenKind::Literal(value),
470 line: start_line,
471 end_line: line,
472 });
473 index = next;
474 }
475 '"' => {
476 let start_line = line;
477 let (next, value) = consume_quoted(&characters, index, '"', false, &mut line);
478 tokens.push(Token {
479 kind: TokenKind::Literal(value),
480 line: start_line,
481 end_line: line,
482 });
483 index = next;
484 }
485 '\'' => {
486 let (next, _) = consume_quoted(&characters, index, '\'', false, &mut line);
487 index = next;
488 }
489 character if is_ident_start(character) => {
490 let start = index;
491 index += 1;
492 while index < characters.len() && is_ident_continue(characters[index]) {
493 index += 1;
494 }
495 tokens.push(Token {
496 kind: TokenKind::Ident(characters[start..index].iter().collect()),
497 line,
498 end_line: line,
499 });
500 }
501 character => {
502 tokens.push(Token {
503 kind: TokenKind::Punct(character),
504 line,
505 end_line: line,
506 });
507 index += 1;
508 }
509 }
510 }
511 tokens
512}
513
514fn rust_raw_string_start(characters: &[char], index: usize) -> Option<usize> {
515 let mut cursor = index + 1;
516 while characters.get(cursor) == Some(&'#') {
517 cursor += 1;
518 }
519 (characters.get(cursor) == Some(&'"')).then_some(cursor - index - 1)
520}
521
522fn consume_rust_raw_string(
523 characters: &[char],
524 index: usize,
525 line: &mut u32,
526) -> (usize, Option<String>) {
527 let hashes = rust_raw_string_start(characters, index).unwrap_or_default();
528 let content_start = index + hashes + 2;
529 let mut cursor = content_start;
530 while cursor < characters.len() {
531 if characters[cursor] == '\n' {
532 *line += 1;
533 }
534 if characters[cursor] == '"'
535 && (0..hashes).all(|offset| characters.get(cursor + 1 + offset) == Some(&'#'))
536 {
537 return (
538 cursor + hashes + 1,
539 Some(characters[content_start..cursor].iter().collect()),
540 );
541 }
542 cursor += 1;
543 }
544 (cursor, None)
545}
546
547fn lex_python(source: &str) -> Vec<Token> {
548 let characters = source.chars().collect::<Vec<_>>();
549 let mut tokens = Vec::new();
550 let mut index = 0;
551 let mut line = 1_u32;
552 while index < characters.len() {
553 match characters[index] {
554 '\n' => {
555 line += 1;
556 index += 1;
557 }
558 character if character.is_whitespace() => index += 1,
559 '#' => {
560 while index < characters.len() && characters[index] != '\n' {
561 index += 1;
562 }
563 }
564 prefix @ ('r' | 'R' | 'u' | 'U')
565 if matches!(characters.get(index + 1), Some('"' | '\'')) =>
566 {
567 let start_line = line;
568 let quote = characters[index + 1];
569 let triple = characters.get(index + 2) == Some("e)
570 && characters.get(index + 3) == Some("e);
571 let (next, value) =
572 consume_quoted(&characters, index + 1, quote, triple, &mut line);
573 let _ = prefix;
574 tokens.push(Token {
575 kind: TokenKind::Literal(value),
576 line: start_line,
577 end_line: line,
578 });
579 index = next;
580 }
581 quote @ ('"' | '\'') => {
582 let start_line = line;
583 let triple = characters.get(index + 1) == Some("e)
584 && characters.get(index + 2) == Some("e);
585 let (next, value) = consume_quoted(&characters, index, quote, triple, &mut line);
586 tokens.push(Token {
587 kind: TokenKind::Literal(value),
588 line: start_line,
589 end_line: line,
590 });
591 index = next;
592 }
593 character if is_ident_start(character) => {
594 let start = index;
595 index += 1;
596 while index < characters.len() && is_ident_continue(characters[index]) {
597 index += 1;
598 }
599 tokens.push(Token {
600 kind: TokenKind::Ident(characters[start..index].iter().collect()),
601 line,
602 end_line: line,
603 });
604 }
605 character => {
606 tokens.push(Token {
607 kind: TokenKind::Punct(character),
608 line,
609 end_line: line,
610 });
611 index += 1;
612 }
613 }
614 }
615 tokens
616}
617
618fn consume_quoted(
619 characters: &[char],
620 index: usize,
621 quote: char,
622 triple: bool,
623 line: &mut u32,
624) -> (usize, Option<String>) {
625 let delimiter_width = if triple { 3 } else { 1 };
626 let mut cursor = index + delimiter_width;
627 let mut value = String::new();
628 let mut valid = true;
629 while cursor < characters.len() {
630 if characters[cursor] == '\n' {
631 *line += 1;
632 if !triple {
633 valid = false;
634 }
635 }
636 let closes = if triple {
637 characters.get(cursor) == Some("e)
638 && characters.get(cursor + 1) == Some("e)
639 && characters.get(cursor + 2) == Some("e)
640 } else {
641 characters.get(cursor) == Some("e)
642 };
643 if closes {
644 return (
645 cursor + delimiter_width,
646 if valid { Some(value) } else { None },
647 );
648 }
649 if characters[cursor] == '\\' && !triple {
650 let Some(escaped) = characters.get(cursor + 1).copied() else {
651 return (characters.len(), None);
652 };
653 match escaped {
654 '\\' => value.push('\\'),
655 '"' => value.push('"'),
656 '\'' => value.push('\''),
657 'n' => value.push('\n'),
658 'r' => value.push('\r'),
659 't' => value.push('\t'),
660 _ => valid = false,
661 }
662 cursor += 2;
663 } else {
664 value.push(characters[cursor]);
665 cursor += 1;
666 }
667 }
668 (cursor, None)
669}
670
671fn is_ident_start(character: char) -> bool {
672 character == '_' || character.is_alphabetic()
673}
674
675fn is_ident_continue(character: char) -> bool {
676 character == '_' || character.is_alphanumeric()
677}
678
679fn matching(tokens: &[Token], open: usize, left: char, right: char) -> Option<usize> {
680 if !tokens.get(open).is_some_and(|token| token.is_punct(left)) {
681 return None;
682 }
683 let mut depth = 0_u32;
684 for (index, token) in tokens.iter().enumerate().skip(open) {
685 if token.is_punct(left) {
686 depth += 1;
687 } else if token.is_punct(right) {
688 depth -= 1;
689 if depth == 0 {
690 return Some(index);
691 }
692 }
693 }
694 None
695}
696
697fn rust_functions(tokens: &[Token]) -> Vec<FunctionSpan> {
698 let mut functions = Vec::new();
699 for (index, token) in tokens.iter().enumerate() {
700 if !token.is_ident("fn") {
701 continue;
702 }
703 let Some(name) = tokens.get(index + 1).and_then(Token::ident) else {
704 continue;
705 };
706 let Some(open) = (index + 2..tokens.len())
707 .find(|candidate| tokens[*candidate].is_punct('{') || tokens[*candidate].is_punct(';'))
708 else {
709 continue;
710 };
711 if tokens[open].is_punct(';') {
712 continue;
713 }
714 let end = matching(tokens, open, '{', '}').unwrap_or(tokens.len().saturating_sub(1));
715 functions.push(FunctionSpan {
716 name: name.to_owned(),
717 start_token: index,
718 end_token: end,
719 });
720 }
721 functions
722}
723
724fn enclosing_symbol(functions: &[FunctionSpan], token_index: usize) -> Option<String> {
725 functions
726 .iter()
727 .filter(|function| function.start_token <= token_index && token_index <= function.end_token)
728 .min_by_key(|function| function.end_token - function.start_token)
729 .map(|function| function.name.clone())
730}
731
732fn parse_rust_attributes(tokens: &[Token], observations: &mut SourceObservationCollector<'_>) {
733 let mut index = 0;
734 while index + 2 < tokens.len() {
735 if !tokens[index].is_punct('#') || !tokens[index + 1].is_punct('[') {
736 index += 1;
737 continue;
738 }
739 let Some(close) = matching(tokens, index + 1, '[', ']') else {
740 break;
741 };
742 let Some((function_index, name, signature_end)) = rust_function_after(tokens, close) else {
743 index = close + 1;
744 continue;
745 };
746 let path = attribute_path(tokens, index + 2, close);
747 let framework = match path.as_slice() {
748 [name] if name == "test" => Some(SourceFramework::RustTest),
749 [first, second] if first == "tokio" && second == "test" => {
750 Some(SourceFramework::TokioTest)
751 }
752 [name] if name == "rstest" => Some(SourceFramework::Rstest),
753 _ => None,
754 };
755 if let Some(framework) = framework {
756 observations.push(confirmed_test(
757 SourceLanguage::Rust,
758 framework,
759 name.clone(),
760 tokens[index].line,
761 tokens[signature_end].end_line,
762 ));
763 }
764 parse_utoipa_attribute(
765 tokens,
766 index,
767 close,
768 function_index,
769 signature_end,
770 &name,
771 observations,
772 );
773 if has_ident(tokens, "actix_web") {
774 parse_actix_attribute(
775 tokens,
776 index,
777 close,
778 function_index,
779 signature_end,
780 name,
781 &path,
782 observations,
783 );
784 }
785 index = close + 1;
786 }
787}
788
789fn parse_utoipa_attribute(
790 tokens: &[Token],
791 start: usize,
792 close: usize,
793 function_index: usize,
794 signature_end: usize,
795 symbol: &str,
796 observations: &mut SourceObservationCollector<'_>,
797) {
798 let Some(utoipa_index) = (start..close).find(|index| tokens[*index].is_ident("utoipa")) else {
799 return;
800 };
801 if !tokens[utoipa_index + 1..close]
802 .iter()
803 .any(|token| token.is_ident("path"))
804 {
805 return;
806 }
807 let method = tokens[utoipa_index..close]
808 .iter()
809 .filter_map(Token::ident)
810 .find_map(canonical_method)
811 .map(str::to_owned);
812 let explicit_path = parse_keyword_string_values(tokens, utoipa_index, close, "path")
813 .into_iter()
814 .next();
815 let context_path = parse_keyword_string_values(tokens, utoipa_index, close, "context_path")
816 .into_iter()
817 .next();
818 let actix = rust_actix_route_between(tokens, close.saturating_add(1), function_index);
819 let method = method.or_else(|| actix.as_ref().and_then(|(method, _)| method.clone()));
820 let path = explicit_path.or_else(|| {
821 let (context, (_, route)) = (context_path?, actix?);
822 Some(normalize_source_http_path(&format!("{context}{route}")))
823 });
824 let lines = SourceLineRange {
825 start: tokens[start].line,
826 end: tokens[signature_end].end_line,
827 };
828 let mut observation = http_from_literal(
829 SourceLanguage::Rust,
830 SourceFramework::Utoipa,
831 SourceRole::Provider,
832 method,
833 path.as_deref(),
834 Some(symbol.to_owned()),
835 lines,
836 true,
837 );
838 if observation.status == SourceEpistemicStatus::Confirmed {
839 observation.confidence = 0.75;
840 observation
841 .warnings
842 .push(SourceWarning::AdvisoryDeclaration);
843 }
844 observations.push(observation);
845}
846
847fn rust_actix_route_between(
848 tokens: &[Token],
849 start: usize,
850 end: usize,
851) -> Option<(Option<String>, String)> {
852 let mut index = start;
853 while index + 3 < end {
854 if !tokens[index].is_punct('#') || !tokens[index + 1].is_punct('[') {
855 index += 1;
856 continue;
857 }
858 let close = matching(tokens, index + 1, '[', ']')?;
859 if close > end {
860 return None;
861 }
862 let method = tokens
863 .get(index + 2)
864 .and_then(Token::ident)
865 .and_then(canonical_method)
866 .map(str::to_owned);
867 let open = (index + 2..close).find(|candidate| tokens[*candidate].is_punct('('));
868 let path = open
869 .and_then(|open| first_argument(open, close))
870 .and_then(|argument| tokens.get(argument))
871 .and_then(Token::literal)
872 .and_then(route_literal_path)?;
873 return Some((method, path));
874 }
875 None
876}
877
878fn rust_function_after(tokens: &[Token], close: usize) -> Option<(usize, String, usize)> {
879 let mut index = close + 1;
880 while index < tokens.len() && index <= close + 24 {
881 if tokens[index].is_punct('#') {
882 if tokens
883 .get(index + 1)
884 .is_some_and(|token| token.is_punct('['))
885 {
886 index = matching(tokens, index + 1, '[', ']')? + 1;
887 continue;
888 }
889 return None;
890 }
891 if tokens[index].is_ident("fn") {
892 let name = tokens.get(index + 1)?.ident()?.to_owned();
893 let end = (index + 2..tokens.len())
894 .find(|candidate| {
895 tokens[*candidate].is_punct('{') || tokens[*candidate].is_punct(';')
896 })
897 .unwrap_or(index + 1);
898 return Some((index, name, end));
899 }
900 index += 1;
901 }
902 None
903}
904
905fn attribute_path(tokens: &[Token], start: usize, end: usize) -> Vec<String> {
906 let mut path = Vec::new();
907 let mut index = start;
908 while index < end && !tokens[index].is_punct('(') {
909 if let Some(name) = tokens[index].ident() {
910 path.push(name.to_owned());
911 }
912 index += 1;
913 }
914 path
915}
916
917#[expect(
918 clippy::too_many_arguments,
919 reason = "Attribute evidence is passed without allocation"
920)]
921fn parse_actix_attribute(
922 tokens: &[Token],
923 start: usize,
924 close: usize,
925 _function_index: usize,
926 signature_end: usize,
927 symbol: String,
928 path: &[String],
929 observations: &mut SourceObservationCollector<'_>,
930) {
931 let direct_method = match path {
932 [name] => canonical_method(name),
933 _ => None,
934 };
935 if direct_method.is_none() && path != ["route"] {
936 return;
937 }
938 let open = (start + 2..close).find(|index| tokens[*index].is_punct('('));
939 let literal = open
940 .and_then(|open| first_argument(open, close))
941 .and_then(|argument| tokens.get(argument))
942 .and_then(Token::literal);
943 let methods = direct_method
944 .into_iter()
945 .map(str::to_owned)
946 .chain(
947 parse_keyword_string_values(tokens, start, close, "method")
948 .into_iter()
949 .filter_map(|method| canonical_method(&method).map(str::to_owned)),
950 )
951 .collect::<BTreeSet<_>>();
952 if methods.is_empty() {
953 observations.push(inexact_http(
954 SourceLanguage::Rust,
955 SourceFramework::ActixWeb,
956 SourceRole::Provider,
957 None,
958 literal.and_then(route_literal_path),
959 Some(symbol),
960 SourceLineRange {
961 start: tokens[start].line,
962 end: tokens[signature_end].end_line,
963 },
964 SourceWarning::DynamicMethod,
965 ));
966 return;
967 }
968 for method in methods {
969 observations.push(http_from_literal(
970 SourceLanguage::Rust,
971 SourceFramework::ActixWeb,
972 SourceRole::Provider,
973 Some(method),
974 literal,
975 Some(symbol.clone()),
976 SourceLineRange {
977 start: tokens[start].line,
978 end: tokens[signature_end].end_line,
979 },
980 true,
981 ));
982 }
983}
984
985fn parse_axum_routes(tokens: &[Token], observations: &mut SourceObservationCollector<'_>) {
986 for (index, token) in tokens.iter().enumerate() {
987 if !token.is_ident("route")
988 || !tokens
989 .get(index.wrapping_sub(1))
990 .is_some_and(|token| token.is_punct('.'))
991 || !tokens
992 .get(index + 1)
993 .is_some_and(|token| token.is_punct('('))
994 {
995 continue;
996 }
997 let Some(close) = matching(tokens, index + 1, '(', ')') else {
998 continue;
999 };
1000 let arguments = top_level_arguments(tokens, index + 1, close);
1001 if arguments.len() < 2 {
1002 continue;
1003 }
1004 let literal = tokens.get(arguments[0]).and_then(Token::literal);
1005 let methods = method_calls(tokens, arguments[1], close);
1006 for (method, handler) in methods {
1007 observations.push(http_from_literal(
1008 SourceLanguage::Rust,
1009 SourceFramework::Axum,
1010 SourceRole::Provider,
1011 Some(method),
1012 literal,
1013 handler,
1014 SourceLineRange {
1015 start: token.line,
1016 end: tokens[close].end_line,
1017 },
1018 true,
1019 ));
1020 }
1021 }
1022}
1023
1024fn parse_actix_builder_routes(tokens: &[Token], observations: &mut SourceObservationCollector<'_>) {
1025 for (index, token) in tokens.iter().enumerate() {
1026 if !token.is_ident("route")
1027 || !tokens
1028 .get(index + 1)
1029 .is_some_and(|token| token.is_punct('('))
1030 {
1031 continue;
1032 }
1033 let Some(close) = matching(tokens, index + 1, '(', ')') else {
1034 continue;
1035 };
1036 let arguments = top_level_arguments(tokens, index + 1, close);
1037 if arguments.len() < 2 || !contains_ident(tokens, arguments[1], close, "web") {
1038 continue;
1039 }
1040 let literal = tokens.get(arguments[0]).and_then(Token::literal);
1041 for (method, handler) in method_calls(tokens, arguments[1], close) {
1042 observations.push(http_from_literal(
1043 SourceLanguage::Rust,
1044 SourceFramework::ActixWeb,
1045 SourceRole::Provider,
1046 Some(method),
1047 literal,
1048 handler,
1049 SourceLineRange {
1050 start: token.line,
1051 end: tokens[close].end_line,
1052 },
1053 true,
1054 ));
1055 }
1056 }
1057 for (index, token) in tokens.iter().enumerate() {
1058 if !token.is_ident("resource")
1059 || !tokens
1060 .get(index.wrapping_sub(1))
1061 .is_some_and(|token| token.is_punct(':'))
1062 || !tokens
1063 .get(index + 1)
1064 .is_some_and(|token| token.is_punct('('))
1065 {
1066 continue;
1067 }
1068 let Some(resource_close) = matching(tokens, index + 1, '(', ')') else {
1069 continue;
1070 };
1071 let end = (resource_close + 1..tokens.len())
1072 .find(|candidate| tokens[*candidate].is_punct(';'))
1073 .unwrap_or(resource_close);
1074 if !contains_ident(tokens, resource_close + 1, end, "route") {
1075 continue;
1076 }
1077 let literal = tokens.get(index + 2).and_then(Token::literal);
1078 for (method, handler) in method_calls(tokens, resource_close + 1, end) {
1079 observations.push(http_from_literal(
1080 SourceLanguage::Rust,
1081 SourceFramework::ActixWeb,
1082 SourceRole::Provider,
1083 Some(method),
1084 literal,
1085 handler,
1086 SourceLineRange {
1087 start: token.line,
1088 end: tokens[end].end_line,
1089 },
1090 true,
1091 ));
1092 }
1093 }
1094}
1095
1096fn parse_reqwest_calls(
1097 tokens: &[Token],
1098 functions: &[FunctionSpan],
1099 observations: &mut SourceObservationCollector<'_>,
1100) {
1101 let clients = rust_reqwest_clients(tokens);
1102 for index in 0..tokens.len() {
1103 let explicit = tokens[index].is_ident("reqwest")
1104 && tokens
1105 .get(index + 1)
1106 .is_some_and(|token| token.is_punct(':'))
1107 && tokens
1108 .get(index + 2)
1109 .is_some_and(|token| token.is_punct(':'));
1110 let (receiver, method_index) = if explicit {
1111 let direct = index + 3;
1112 let method_index = if tokens
1113 .get(direct)
1114 .is_some_and(|token| token.is_ident("blocking"))
1115 && tokens
1116 .get(direct + 1)
1117 .is_some_and(|token| token.is_punct(':'))
1118 && tokens
1119 .get(direct + 2)
1120 .is_some_and(|token| token.is_punct(':'))
1121 {
1122 direct + 3
1123 } else {
1124 direct
1125 };
1126 ("reqwest", method_index)
1127 } else if let Some(receiver) = tokens[index].ident() {
1128 if clients.contains(receiver)
1129 && tokens
1130 .get(index + 1)
1131 .is_some_and(|token| token.is_punct('.'))
1132 {
1133 (receiver, index + 2)
1134 } else {
1135 continue;
1136 }
1137 } else {
1138 continue;
1139 };
1140 let Some(method_name) = tokens.get(method_index).and_then(Token::ident) else {
1141 continue;
1142 };
1143 if !tokens
1144 .get(method_index + 1)
1145 .is_some_and(|token| token.is_punct('('))
1146 {
1147 continue;
1148 }
1149 let Some(close) = matching(tokens, method_index + 1, '(', ')') else {
1150 continue;
1151 };
1152 let arguments = top_level_arguments(tokens, method_index + 1, close);
1153 let (method, path_argument) = if method_name == "request" {
1154 (
1155 arguments.first().and_then(|argument| {
1156 let method_end = arguments
1157 .get(1)
1158 .map_or(close, |second| second.saturating_sub(1));
1159 rust_method_expression(tokens, *argument, method_end)
1160 }),
1161 arguments.get(1).copied(),
1162 )
1163 } else {
1164 (
1165 canonical_method(method_name).map(str::to_owned),
1166 arguments.first().copied(),
1167 )
1168 };
1169 if method.is_none() && method_name != "request" {
1170 continue;
1171 }
1172 let literal = path_argument
1173 .and_then(|argument| tokens.get(argument))
1174 .and_then(Token::literal);
1175 let mut observation = http_from_literal(
1176 SourceLanguage::Rust,
1177 SourceFramework::Reqwest,
1178 SourceRole::Consumer,
1179 method,
1180 literal,
1181 enclosing_symbol(functions, index),
1182 SourceLineRange {
1183 start: tokens[index].line,
1184 end: tokens[close].end_line,
1185 },
1186 false,
1187 );
1188 if method_name == "request" && observation.method.is_none() {
1189 observation.status = SourceEpistemicStatus::Incomplete;
1190 observation.confidence = 0.0;
1191 observation.warnings.push(SourceWarning::DynamicMethod);
1192 observation.warnings.sort();
1193 observation.warnings.dedup();
1194 }
1195 let _ = receiver;
1196 observations.push(observation);
1197 }
1198}
1199
1200fn rust_reqwest_clients(tokens: &[Token]) -> BTreeSet<String> {
1201 let mut clients = BTreeSet::new();
1202 let client_imported = tokens.iter().enumerate().any(|(index, token)| {
1203 token.is_ident("reqwest")
1204 && (index + 1..tokens.len())
1205 .take_while(|candidate| !tokens[*candidate].is_punct(';'))
1206 .take(24)
1207 .any(|candidate| tokens[candidate].is_ident("Client"))
1208 });
1209 for index in 0..tokens.len() {
1210 if !tokens[index].is_ident("let") {
1211 continue;
1212 }
1213 let Some(name) = tokens.get(index + 1).and_then(Token::ident) else {
1214 continue;
1215 };
1216 let end = (index + 2..tokens.len())
1217 .find(|candidate| tokens[*candidate].is_punct(';'))
1218 .unwrap_or(tokens.len());
1219 let uses_qualified_client = contains_ident(tokens, index + 2, end, "reqwest")
1220 && contains_ident(tokens, index + 2, end, "Client");
1221 let uses_imported_client = client_imported
1222 && tokens
1223 .get(index + 3)
1224 .is_some_and(|token| token.is_ident("Client"));
1225 if uses_qualified_client || uses_imported_client {
1226 clients.insert(name.to_owned());
1227 }
1228 }
1229 clients
1230}
1231
1232fn rust_method_expression(tokens: &[Token], start: usize, end: usize) -> Option<String> {
1233 if let Some(literal) = tokens.get(start).and_then(Token::literal) {
1234 return canonical_method(literal).map(str::to_owned);
1235 }
1236 (start..end)
1237 .filter_map(|index| tokens[index].ident())
1238 .find_map(|name| canonical_method(name).map(str::to_owned))
1239}
1240
1241fn method_calls(tokens: &[Token], start: usize, end: usize) -> Vec<(String, Option<String>)> {
1242 let mut methods = Vec::new();
1243 for index in start..end {
1244 let Some(name) = tokens[index].ident() else {
1245 continue;
1246 };
1247 let Some(method) = canonical_method(name) else {
1248 continue;
1249 };
1250 if !tokens
1251 .get(index + 1)
1252 .is_some_and(|token| token.is_punct('('))
1253 {
1254 continue;
1255 }
1256 let direct_handler = tokens
1257 .get(index + 2)
1258 .and_then(Token::ident)
1259 .map(str::to_owned);
1260 let actix_handler = (index + 2..end)
1261 .find(|candidate| {
1262 tokens[*candidate].is_ident("to")
1263 && tokens
1264 .get(*candidate + 1)
1265 .is_some_and(|token| token.is_punct('('))
1266 })
1267 .and_then(|to| tokens.get(to + 2))
1268 .and_then(Token::ident)
1269 .map(str::to_owned);
1270 let handler = direct_handler.or(actix_handler);
1271 methods.push((method.to_owned(), handler));
1272 }
1273 methods
1274}
1275
1276fn contains_ident(tokens: &[Token], start: usize, end: usize, name: &str) -> bool {
1277 tokens
1278 .get(start..end)
1279 .is_some_and(|slice| slice.iter().any(|token| token.is_ident(name)))
1280}
1281
1282fn first_argument(open: usize, close: usize) -> Option<usize> {
1283 (open + 1 < close).then_some(open + 1)
1284}
1285
1286fn top_level_arguments(tokens: &[Token], open: usize, close: usize) -> Vec<usize> {
1287 if open + 1 >= close {
1288 return Vec::new();
1289 }
1290 let mut arguments = vec![open + 1];
1291 let mut round = 0_i32;
1292 let mut square = 0_i32;
1293 let mut curly = 0_i32;
1294 for (index, token) in tokens.iter().enumerate().take(close).skip(open + 1) {
1295 match token.kind {
1296 TokenKind::Punct('(') => round += 1,
1297 TokenKind::Punct(')') => round -= 1,
1298 TokenKind::Punct('[') => square += 1,
1299 TokenKind::Punct(']') => square -= 1,
1300 TokenKind::Punct('{') => curly += 1,
1301 TokenKind::Punct('}') => curly -= 1,
1302 TokenKind::Punct(',')
1303 if round == 0 && square == 0 && curly == 0 && index + 1 < close =>
1304 {
1305 arguments.push(index + 1);
1306 }
1307 _ => {}
1308 }
1309 }
1310 arguments
1311}
1312
1313fn parse_keyword_string_values(
1314 tokens: &[Token],
1315 start: usize,
1316 end: usize,
1317 keyword: &str,
1318) -> Vec<String> {
1319 let mut values = Vec::new();
1320 for index in start..end {
1321 if !tokens[index].is_ident(keyword)
1322 || !tokens
1323 .get(index + 1)
1324 .is_some_and(|token| token.is_punct('='))
1325 {
1326 continue;
1327 }
1328 let value_start = index + 2;
1329 if let Some(value) = tokens.get(value_start).and_then(Token::literal) {
1330 values.push(value.to_owned());
1331 } else if tokens
1332 .get(value_start)
1333 .is_some_and(|token| token.is_punct('['))
1334 {
1335 let close = matching(tokens, value_start, '[', ']').unwrap_or(end);
1336 values.extend(
1337 tokens[value_start + 1..close]
1338 .iter()
1339 .filter_map(Token::literal)
1340 .map(str::to_owned),
1341 );
1342 }
1343 }
1344 values
1345}
1346
1347fn canonical_method(name: &str) -> Option<&'static str> {
1348 HTTP_METHODS
1349 .iter()
1350 .copied()
1351 .find(|method| method.eq_ignore_ascii_case(name))
1352}
1353
1354fn confirmed_test(
1355 language: SourceLanguage,
1356 framework: SourceFramework,
1357 name: String,
1358 start: u32,
1359 end: u32,
1360) -> SourceObservation {
1361 SourceObservation {
1362 language,
1363 framework,
1364 role: SourceRole::Test,
1365 method: None,
1366 path: None,
1367 symbol_name: Some(name),
1368 related_symbol: None,
1369 related_path: None,
1370 lines: SourceLineRange { start, end },
1371 status: SourceEpistemicStatus::Confirmed,
1372 confidence: 1.0,
1373 warnings: Vec::new(),
1374 }
1375}
1376
1377#[expect(
1378 clippy::too_many_arguments,
1379 reason = "Observation fields remain explicit at evidence sites"
1380)]
1381fn http_from_literal(
1382 language: SourceLanguage,
1383 framework: SourceFramework,
1384 role: SourceRole,
1385 method: Option<String>,
1386 literal: Option<&str>,
1387 symbol_name: Option<String>,
1388 lines: SourceLineRange,
1389 route: bool,
1390) -> SourceObservation {
1391 let path = literal.and_then(|value| {
1392 if route {
1393 route_literal_path(value)
1394 } else {
1395 client_literal_path(value)
1396 }
1397 });
1398 let warning = if literal.is_none() {
1399 Some(SourceWarning::DynamicPath)
1400 } else if path.is_none() {
1401 Some(SourceWarning::UnsupportedLiteralPath)
1402 } else {
1403 None
1404 };
1405 let status = if path.is_some() && method.is_some() {
1406 SourceEpistemicStatus::Confirmed
1407 } else {
1408 SourceEpistemicStatus::Ambiguous
1409 };
1410 let mut observation = SourceObservation {
1411 language,
1412 framework,
1413 role,
1414 method,
1415 path,
1416 symbol_name,
1417 related_symbol: None,
1418 related_path: None,
1419 lines,
1420 status,
1421 confidence: if status == SourceEpistemicStatus::Confirmed {
1422 1.0
1423 } else {
1424 0.0
1425 },
1426 warnings: warning.into_iter().collect(),
1427 };
1428 if role == SourceRole::Provider && observation.symbol_name.is_none() {
1429 observation.status = SourceEpistemicStatus::Incomplete;
1430 observation.confidence = 0.0;
1431 observation.warnings.push(SourceWarning::MissingSymbol);
1432 }
1433 observation
1434}
1435
1436#[expect(
1437 clippy::too_many_arguments,
1438 reason = "Observation fields remain explicit at evidence sites"
1439)]
1440fn inexact_http(
1441 language: SourceLanguage,
1442 framework: SourceFramework,
1443 role: SourceRole,
1444 method: Option<String>,
1445 path: Option<String>,
1446 symbol_name: Option<String>,
1447 lines: SourceLineRange,
1448 warning: SourceWarning,
1449) -> SourceObservation {
1450 SourceObservation {
1451 language,
1452 framework,
1453 role,
1454 method,
1455 path,
1456 symbol_name,
1457 related_symbol: None,
1458 related_path: None,
1459 lines,
1460 status: SourceEpistemicStatus::Incomplete,
1461 confidence: 0.0,
1462 warnings: vec![warning],
1463 }
1464}
1465
1466fn route_literal_path(value: &str) -> Option<String> {
1467 (value.starts_with('/') && !value.contains(['?', '#']))
1468 .then(|| normalize_source_http_path(value))
1469}
1470
1471fn client_literal_path(value: &str) -> Option<String> {
1472 if value.starts_with('/') {
1473 return Some(normalize_source_http_path(
1474 value.split(['?', '#']).next().unwrap_or(value),
1475 ));
1476 }
1477 let after_authority = value
1478 .strip_prefix("http://")
1479 .or_else(|| value.strip_prefix("https://"))?;
1480 if after_authority.is_empty() || after_authority.starts_with('/') {
1481 return None;
1482 }
1483 let path = after_authority
1484 .find('/')
1485 .map_or("/", |separator| &after_authority[separator..]);
1486 Some(normalize_source_http_path(
1487 path.split(['?', '#']).next().unwrap_or(path),
1488 ))
1489}
1490
1491#[derive(Debug, Default)]
1492struct PythonContexts {
1493 fastapi_apps: BTreeSet<String>,
1494 route_prefixes: BTreeMap<String, String>,
1495 flask_apps: BTreeSet<String>,
1496 request_modules: BTreeSet<String>,
1497 httpx_modules: BTreeSet<String>,
1498 aiohttp_modules: BTreeSet<String>,
1499 request_clients: BTreeSet<String>,
1500 httpx_clients: BTreeSet<String>,
1501 aiohttp_clients: BTreeSet<String>,
1502 direct_calls: BTreeMap<String, (SourceFramework, String)>,
1503 string_constants: BTreeMap<String, String>,
1504}
1505
1506impl PythonContexts {
1507 fn discover(tokens: &[Token]) -> Self {
1508 let mut contexts = Self::default();
1509 contexts.request_modules.insert("requests".to_owned());
1510 contexts.httpx_modules.insert("httpx".to_owned());
1511 contexts.aiohttp_modules.insert("aiohttp".to_owned());
1512 discover_python_import_aliases(tokens, "requests", &mut contexts.request_modules);
1513 discover_python_import_aliases(tokens, "httpx", &mut contexts.httpx_modules);
1514 discover_python_import_aliases(tokens, "aiohttp", &mut contexts.aiohttp_modules);
1515 discover_python_direct_calls(
1516 tokens,
1517 "requests",
1518 SourceFramework::Requests,
1519 &mut contexts.direct_calls,
1520 );
1521 discover_python_direct_calls(
1522 tokens,
1523 "httpx",
1524 SourceFramework::Httpx,
1525 &mut contexts.direct_calls,
1526 );
1527 let request_session_imported = python_imports_item(tokens, "requests", "Session");
1528 let httpx_client_imported = python_imports_item(tokens, "httpx", "Client")
1529 || python_imports_item(tokens, "httpx", "AsyncClient");
1530 let aiohttp_client_imported = python_imports_item(tokens, "aiohttp", "ClientSession");
1531 for index in 0..tokens.len() {
1532 let Some(name) = tokens[index].ident() else {
1533 continue;
1534 };
1535 if !tokens
1536 .get(index + 1)
1537 .is_some_and(|token| token.is_punct('='))
1538 {
1539 continue;
1540 }
1541 let end = python_expression_end(tokens, index + 2);
1542 if let Some(value) =
1543 python_static_string_expression(tokens, index + 2, end, &contexts.string_constants)
1544 {
1545 contexts.string_constants.insert(name.to_owned(), value);
1546 }
1547 if contains_ident(tokens, index + 2, end, "FastAPI")
1548 || contains_ident(tokens, index + 2, end, "APIRouter")
1549 {
1550 contexts.fastapi_apps.insert(name.to_owned());
1551 if contains_ident(tokens, index + 2, end, "APIRouter")
1552 && let Some(open) =
1553 (index + 2..end).find(|candidate| tokens[*candidate].is_punct('('))
1554 && let Some(close) = matching(tokens, open, '(', ')')
1555 && let Some(prefix) = parse_keyword_string_values(tokens, open, close, "prefix")
1556 .into_iter()
1557 .next()
1558 {
1559 contexts
1560 .route_prefixes
1561 .insert(name.to_owned(), normalize_source_http_path(&prefix));
1562 }
1563 }
1564 if contains_ident(tokens, index + 2, end, "Flask")
1565 || contains_ident(tokens, index + 2, end, "Blueprint")
1566 {
1567 contexts.flask_apps.insert(name.to_owned());
1568 }
1569 if contains_ident(tokens, index + 2, end, "Session")
1570 && (contains_any(tokens, index + 2, end, &contexts.request_modules)
1571 || request_session_imported)
1572 {
1573 contexts.request_clients.insert(name.to_owned());
1574 }
1575 if (contains_ident(tokens, index + 2, end, "Client")
1576 || contains_ident(tokens, index + 2, end, "AsyncClient"))
1577 && (contains_any(tokens, index + 2, end, &contexts.httpx_modules)
1578 || httpx_client_imported)
1579 {
1580 contexts.httpx_clients.insert(name.to_owned());
1581 }
1582 if contains_ident(tokens, index + 2, end, "ClientSession")
1583 && (contains_any(tokens, index + 2, end, &contexts.aiohttp_modules)
1584 || aiohttp_client_imported)
1585 {
1586 contexts.aiohttp_clients.insert(name.to_owned());
1587 }
1588 }
1589 if aiohttp_client_imported {
1590 for index in 0..tokens.len() {
1591 if !tokens[index].is_ident("as") {
1592 continue;
1593 }
1594 let Some(name) = tokens.get(index + 1).and_then(Token::ident) else {
1595 continue;
1596 };
1597 let start = index.saturating_sub(32);
1598 if tokens[start..index]
1599 .iter()
1600 .any(|token| token.is_ident("ClientSession"))
1601 {
1602 contexts.aiohttp_clients.insert(name.to_owned());
1603 }
1604 }
1605 }
1606 contexts
1607 }
1608}
1609
1610fn python_imports_item(tokens: &[Token], module: &str, item: &str) -> bool {
1611 tokens.iter().enumerate().any(|(index, token)| {
1612 token.is_ident("from")
1613 && tokens
1614 .get(index + 1)
1615 .is_some_and(|token| token.is_ident(module))
1616 && tokens
1617 .get(index + 2)
1618 .is_some_and(|token| token.is_ident("import"))
1619 && (index + 3..tokens.len())
1620 .take_while(|candidate| tokens[*candidate].line == token.line)
1621 .any(|candidate| tokens[candidate].is_ident(item))
1622 })
1623}
1624
1625fn discover_python_direct_calls(
1626 tokens: &[Token],
1627 module: &str,
1628 framework: SourceFramework,
1629 calls: &mut BTreeMap<String, (SourceFramework, String)>,
1630) {
1631 for index in 0..tokens.len() {
1632 if !tokens[index].is_ident("from")
1633 || !tokens
1634 .get(index + 1)
1635 .is_some_and(|token| token.is_ident(module))
1636 || !tokens
1637 .get(index + 2)
1638 .is_some_and(|token| token.is_ident("import"))
1639 {
1640 continue;
1641 }
1642 let mut cursor = index + 3;
1643 while cursor < tokens.len() && tokens[cursor].line == tokens[index].line {
1644 let Some(imported) = tokens[cursor].ident() else {
1645 cursor += 1;
1646 continue;
1647 };
1648 if canonical_method(imported).is_none() && imported != "request" {
1649 cursor += 1;
1650 continue;
1651 }
1652 let (local, advance) = if tokens
1653 .get(cursor + 1)
1654 .is_some_and(|token| token.is_ident("as"))
1655 {
1656 (
1657 tokens
1658 .get(cursor + 2)
1659 .and_then(Token::ident)
1660 .unwrap_or(imported),
1661 3,
1662 )
1663 } else {
1664 (imported, 1)
1665 };
1666 calls.insert(local.to_owned(), (framework, imported.to_owned()));
1667 cursor += advance;
1668 }
1669 }
1670}
1671
1672fn python_expression_end(tokens: &[Token], start: usize) -> usize {
1673 let mut round = 0_i32;
1674 let mut square = 0_i32;
1675 let mut curly = 0_i32;
1676 let mut previous_line = tokens.get(start).map_or(0, |token| token.line);
1677 for (index, token) in tokens.iter().enumerate().skip(start) {
1678 if index > start && token.line > previous_line && round == 0 && square == 0 && curly == 0 {
1679 return index;
1680 }
1681 if token.is_punct(';') && round == 0 && square == 0 && curly == 0 {
1682 return index;
1683 }
1684 match token.kind {
1685 TokenKind::Punct('(') => round += 1,
1686 TokenKind::Punct(')') => round -= 1,
1687 TokenKind::Punct('[') => square += 1,
1688 TokenKind::Punct(']') => square -= 1,
1689 TokenKind::Punct('{') => curly += 1,
1690 TokenKind::Punct('}') => curly -= 1,
1691 _ => {}
1692 }
1693 previous_line = token.line;
1694 }
1695 tokens.len()
1696}
1697
1698fn discover_python_import_aliases(tokens: &[Token], module: &str, aliases: &mut BTreeSet<String>) {
1699 for index in 0..tokens.len() {
1700 if !tokens[index].is_ident("import")
1701 || !tokens
1702 .get(index + 1)
1703 .is_some_and(|token| token.is_ident(module))
1704 {
1705 continue;
1706 }
1707 if tokens
1708 .get(index + 2)
1709 .is_some_and(|token| token.is_ident("as"))
1710 && let Some(alias) = tokens.get(index + 3).and_then(Token::ident)
1711 {
1712 aliases.insert(alias.to_owned());
1713 }
1714 }
1715}
1716
1717fn contains_any(tokens: &[Token], start: usize, end: usize, names: &BTreeSet<String>) -> bool {
1718 tokens.get(start..end).is_some_and(|slice| {
1719 slice
1720 .iter()
1721 .filter_map(Token::ident)
1722 .any(|name| names.contains(name))
1723 })
1724}
1725
1726fn python_static_string_expression(
1727 tokens: &[Token],
1728 start: usize,
1729 end: usize,
1730 constants: &BTreeMap<String, String>,
1731) -> Option<String> {
1732 let mut output = String::new();
1733 let mut found = false;
1734 for token in tokens.get(start..end)? {
1735 if let Some(value) = token.literal() {
1736 output.push_str(value);
1737 found = true;
1738 } else if let Some(name) = token.ident() {
1739 output.push_str(constants.get(name)?);
1740 found = true;
1741 } else if !matches!(token.kind, TokenKind::Punct('+' | '(' | ')' | '[' | ']')) {
1742 return None;
1743 }
1744 }
1745 found.then_some(output)
1746}
1747
1748fn python_functions(source: &str, tokens: &[Token]) -> Vec<FunctionSpan> {
1749 let line_count = saturating_u32(source.lines().count().max(1));
1750 let mut functions = Vec::new();
1751 for index in 0..tokens.len() {
1752 if !tokens[index].is_ident("def") {
1753 continue;
1754 }
1755 let Some(name) = tokens.get(index + 1).and_then(Token::ident) else {
1756 continue;
1757 };
1758 let indent = source_indent(source, tokens[index].line);
1759 let end_line = python_block_end(source, tokens[index].line, indent, line_count);
1760 let end_token = (index + 1..tokens.len())
1761 .find(|candidate| tokens[*candidate].line > end_line)
1762 .unwrap_or(tokens.len())
1763 .saturating_sub(1);
1764 functions.push(FunctionSpan {
1765 name: name.to_owned(),
1766 start_token: index,
1767 end_token,
1768 });
1769 }
1770 functions
1771}
1772
1773fn source_indent(source: &str, line: u32) -> usize {
1774 source
1775 .lines()
1776 .nth(line.saturating_sub(1) as usize)
1777 .map_or(0, |value| {
1778 value
1779 .chars()
1780 .take_while(|character| character.is_whitespace())
1781 .count()
1782 })
1783}
1784
1785fn python_block_end(source: &str, start: u32, indent: usize, fallback: u32) -> u32 {
1786 for (offset, line) in source.lines().enumerate().skip(start as usize) {
1787 let trimmed = line.trim();
1788 if trimmed.is_empty() || trimmed.starts_with('#') {
1789 continue;
1790 }
1791 let candidate_indent = line
1792 .chars()
1793 .take_while(|character| character.is_whitespace())
1794 .count();
1795 if candidate_indent <= indent {
1796 return saturating_u32(offset);
1797 }
1798 }
1799 fallback
1800}
1801
1802fn parse_python_routes(
1803 tokens: &[Token],
1804 contexts: &PythonContexts,
1805 observations: &mut SourceObservationCollector<'_>,
1806) {
1807 for index in 0..tokens.len() {
1808 if !tokens[index].is_punct('@') {
1809 continue;
1810 }
1811 let Some((receiver, decorator, open)) = python_decorator_call(tokens, index) else {
1812 continue;
1813 };
1814 let Some(close) = matching(tokens, open, '(', ')') else {
1815 continue;
1816 };
1817 let Some((name, signature_end)) = python_function_after(tokens, close) else {
1818 continue;
1819 };
1820 let framework = if contexts.fastapi_apps.contains(receiver) {
1821 SourceFramework::FastApi
1822 } else if contexts.flask_apps.contains(receiver) {
1823 SourceFramework::Flask
1824 } else {
1825 continue;
1826 };
1827 let path_start = top_level_arguments(tokens, open, close).first().copied();
1828 let resolved_path = path_start.and_then(|start| {
1829 python_static_string_expression(
1830 tokens,
1831 start,
1832 python_argument_end(tokens, start, close),
1833 &contexts.string_constants,
1834 )
1835 });
1836 let raw_path = resolved_path.or_else(|| {
1837 tokens
1838 .get(open + 1)
1839 .and_then(Token::literal)
1840 .map(str::to_owned)
1841 });
1842 let prefixed_path = raw_path.as_ref().map(|path| {
1843 contexts.route_prefixes.get(receiver).map_or_else(
1844 || path.clone(),
1845 |prefix| normalize_source_http_path(&format!("{prefix}{path}")),
1846 )
1847 });
1848 let literal = prefixed_path.as_deref();
1849 let methods = python_route_methods(tokens, open, close, decorator, framework);
1850 if methods.is_empty() {
1851 observations.push(inexact_http(
1852 SourceLanguage::Python,
1853 framework,
1854 SourceRole::Provider,
1855 None,
1856 literal.and_then(route_literal_path),
1857 Some(name),
1858 SourceLineRange {
1859 start: tokens[index].line,
1860 end: tokens[signature_end].end_line,
1861 },
1862 SourceWarning::DynamicMethod,
1863 ));
1864 } else {
1865 for method in methods {
1866 observations.push(http_from_literal(
1867 SourceLanguage::Python,
1868 framework,
1869 SourceRole::Provider,
1870 Some(method),
1871 literal,
1872 Some(name.clone()),
1873 SourceLineRange {
1874 start: tokens[index].line,
1875 end: tokens[signature_end].end_line,
1876 },
1877 true,
1878 ));
1879 }
1880 }
1881 }
1882}
1883
1884fn python_decorator_call(tokens: &[Token], at: usize) -> Option<(&str, &str, usize)> {
1885 let mut names = Vec::new();
1886 let mut cursor = at.saturating_add(1);
1887 loop {
1888 names.push(tokens.get(cursor)?.ident()?);
1889 if tokens
1890 .get(cursor + 1)
1891 .is_some_and(|token| token.is_punct('.'))
1892 {
1893 cursor = cursor.saturating_add(2);
1894 continue;
1895 }
1896 break;
1897 }
1898 let open = cursor.saturating_add(1);
1899 if names.len() < 2 || !tokens.get(open).is_some_and(|token| token.is_punct('(')) {
1900 return None;
1901 }
1902 Some((names[names.len() - 2], names[names.len() - 1], open))
1903}
1904
1905fn python_argument_end(tokens: &[Token], start: usize, close: usize) -> usize {
1906 let mut round = 0_i32;
1907 let mut square = 0_i32;
1908 let mut curly = 0_i32;
1909 for (index, token) in tokens.iter().enumerate().take(close).skip(start) {
1910 match token.kind {
1911 TokenKind::Punct('(') => round += 1,
1912 TokenKind::Punct(')') => round -= 1,
1913 TokenKind::Punct('[') => square += 1,
1914 TokenKind::Punct(']') => square -= 1,
1915 TokenKind::Punct('{') => curly += 1,
1916 TokenKind::Punct('}') => curly -= 1,
1917 TokenKind::Punct(',') if round == 0 && square == 0 && curly == 0 => return index,
1918 _ => {}
1919 }
1920 }
1921 close
1922}
1923
1924fn python_function_after(tokens: &[Token], close: usize) -> Option<(String, usize)> {
1925 let mut index = close + 1;
1926 while index < tokens.len() && tokens[index].line <= tokens[close].line.saturating_add(12) {
1927 if tokens[index].is_ident("async") {
1928 index += 1;
1929 continue;
1930 }
1931 if tokens[index].is_ident("def") {
1932 let name = tokens.get(index + 1)?.ident()?.to_owned();
1933 let end = (index + 2..tokens.len())
1934 .find(|candidate| tokens[*candidate].is_punct(':'))
1935 .unwrap_or(index + 1);
1936 return Some((name, end));
1937 }
1938 if tokens[index].is_punct('@') {
1939 return None;
1940 }
1941 index += 1;
1942 }
1943 None
1944}
1945
1946fn python_route_methods(
1947 tokens: &[Token],
1948 open: usize,
1949 close: usize,
1950 decorator: &str,
1951 framework: SourceFramework,
1952) -> BTreeSet<String> {
1953 if let Some(method) = canonical_method(decorator) {
1954 return BTreeSet::from([method.to_owned()]);
1955 }
1956 let mut methods = parse_keyword_string_values(tokens, open, close, "methods")
1957 .into_iter()
1958 .filter_map(|method| canonical_method(&method).map(str::to_owned))
1959 .collect::<BTreeSet<_>>();
1960 if framework == SourceFramework::Flask && decorator == "route" && methods.is_empty() {
1961 methods.insert("GET".to_owned());
1962 }
1963 methods
1964}
1965
1966fn parse_python_http_registries(
1967 tokens: &[Token],
1968 observations: &mut SourceObservationCollector<'_>,
1969) {
1970 for index in 0..tokens.len() {
1971 let Some(name) = tokens[index].ident() else {
1972 continue;
1973 };
1974 if !name
1975 .chars()
1976 .all(|character| character.is_ascii_uppercase() || character == '_')
1977 || !tokens
1978 .get(index + 1)
1979 .is_some_and(|token| token.is_punct('='))
1980 || !tokens
1981 .get(index + 2)
1982 .is_some_and(|token| token.is_punct('{'))
1983 {
1984 continue;
1985 }
1986 let open = index + 2;
1987 let Some(close) = matching(tokens, open, '{', '}') else {
1988 continue;
1989 };
1990 parse_python_http_registry_dict(tokens, open, close, "", observations, 0);
1991 }
1992}
1993
1994fn parse_python_http_registry_dict(
1995 tokens: &[Token],
1996 open: usize,
1997 close: usize,
1998 prefix: &str,
1999 observations: &mut SourceObservationCollector<'_>,
2000 depth: usize,
2001) {
2002 if depth >= 32 {
2003 return;
2004 }
2005 let mut cursor = open.saturating_add(1);
2006 while cursor < close {
2007 while cursor < close && tokens[cursor].is_punct(',') {
2008 cursor += 1;
2009 }
2010 let Some(key) = tokens.get(cursor).and_then(Token::literal) else {
2011 cursor += 1;
2012 continue;
2013 };
2014 if !tokens
2015 .get(cursor + 1)
2016 .is_some_and(|token| token.is_punct(':'))
2017 {
2018 cursor += 1;
2019 continue;
2020 }
2021 let value_start = cursor + 2;
2022 let value_end = python_argument_end(tokens, value_start, close);
2023 parse_python_http_registry_value(
2024 tokens,
2025 value_start,
2026 value_end,
2027 prefix,
2028 key,
2029 observations,
2030 depth,
2031 );
2032 cursor = value_end.saturating_add(1);
2033 }
2034}
2035
2036fn parse_python_http_registry_value(
2037 tokens: &[Token],
2038 start: usize,
2039 end: usize,
2040 prefix: &str,
2041 key: &str,
2042 observations: &mut SourceObservationCollector<'_>,
2043 depth: usize,
2044) {
2045 if tokens.get(start).is_some_and(|token| token.is_punct('[')) {
2046 let Some(close) = matching(tokens, start, '[', ']').filter(|close| *close <= end) else {
2047 return;
2048 };
2049 let arguments = top_level_arguments(tokens, start, close);
2050 let Some(method) = arguments
2051 .first()
2052 .and_then(|argument| tokens.get(*argument))
2053 .and_then(Token::literal)
2054 .and_then(canonical_method)
2055 else {
2056 return;
2057 };
2058 let Some(path) = arguments
2059 .get(1)
2060 .and_then(|argument| tokens.get(*argument))
2061 .and_then(Token::literal)
2062 else {
2063 return;
2064 };
2065 let combined = canonical_python_registry_path(prefix, path);
2066 observations.push(http_from_literal(
2067 SourceLanguage::Python,
2068 SourceFramework::PythonHttpRegistry,
2069 SourceRole::Consumer,
2070 Some(method.to_owned()),
2071 Some(&combined),
2072 Some(key.to_owned()),
2073 SourceLineRange {
2074 start: tokens[start].line,
2075 end: tokens[close].end_line,
2076 },
2077 false,
2078 ));
2079 return;
2080 }
2081 if !tokens.get(start).is_some_and(|token| token.is_punct('(')) {
2082 return;
2083 }
2084 let Some(close) = matching(tokens, start, '(', ')').filter(|close| *close <= end) else {
2085 return;
2086 };
2087 let arguments = top_level_arguments(tokens, start, close);
2088 let Some(segment) = arguments
2089 .first()
2090 .and_then(|argument| tokens.get(*argument))
2091 .and_then(Token::literal)
2092 else {
2093 return;
2094 };
2095 let Some(dictionary) = arguments.get(1).copied().filter(|argument| {
2096 tokens
2097 .get(*argument)
2098 .is_some_and(|token| token.is_punct('{'))
2099 }) else {
2100 return;
2101 };
2102 let Some(dictionary_close) = matching(tokens, dictionary, '{', '}') else {
2103 return;
2104 };
2105 let nested_prefix = canonical_python_registry_path(prefix, segment);
2106 parse_python_http_registry_dict(
2107 tokens,
2108 dictionary,
2109 dictionary_close,
2110 &nested_prefix,
2111 observations,
2112 depth.saturating_add(1),
2113 );
2114}
2115
2116fn canonical_python_registry_path(prefix: &str, segment: &str) -> String {
2117 let joined = format!("{prefix}{segment}")
2118 .replace("{{", "{")
2119 .replace("}}", "}");
2120 normalize_source_http_path(&joined)
2121}
2122
2123fn parse_python_factories(
2124 source: &str,
2125 tokens: &[Token],
2126 observations: &mut SourceObservationCollector<'_>,
2127) {
2128 let imported_paths = python_imported_symbol_paths(tokens);
2129 let fallback = saturating_u32(source.lines().count().max(1));
2130 for index in 0..tokens.len() {
2131 if !tokens[index].is_ident("class") {
2132 continue;
2133 }
2134 let Some(factory_name) = tokens.get(index + 1).and_then(Token::ident) else {
2135 continue;
2136 };
2137 let signature_end = (index + 2..tokens.len())
2138 .find(|candidate| tokens[*candidate].is_punct(':'))
2139 .unwrap_or(index + 1);
2140 if !tokens
2141 .get(index + 2..signature_end)
2142 .is_some_and(|signature| {
2143 signature
2144 .iter()
2145 .filter_map(Token::ident)
2146 .any(|base| base == "Factory" || base.ends_with("Factory"))
2147 })
2148 {
2149 continue;
2150 }
2151 let start_line = tokens[index].line;
2152 let end_line = python_block_end(
2153 source,
2154 start_line,
2155 source_indent(source, start_line),
2156 fallback,
2157 );
2158 let model_assignment = (signature_end + 1..tokens.len())
2159 .take_while(|candidate| tokens[*candidate].line <= end_line)
2160 .find_map(|candidate| {
2161 if tokens[candidate].is_ident("model")
2162 && tokens
2163 .get(candidate + 1)
2164 .is_some_and(|token| token.is_punct('='))
2165 {
2166 Some((
2167 tokens.get(candidate + 2).and_then(Token::ident)?,
2168 tokens[candidate].line,
2169 ))
2170 } else {
2171 None
2172 }
2173 });
2174 let Some((model_name, model_line)) = model_assignment else {
2175 continue;
2176 };
2177 observations.push(SourceObservation {
2178 language: SourceLanguage::Python,
2179 framework: SourceFramework::FactoryBoy,
2180 role: SourceRole::Factory,
2181 method: None,
2182 path: None,
2183 symbol_name: Some(factory_name.to_owned()),
2184 related_symbol: Some(model_name.to_owned()),
2185 related_path: imported_paths.get(model_name).cloned(),
2186 lines: SourceLineRange {
2187 start: start_line,
2188 end: model_line,
2189 },
2190 status: SourceEpistemicStatus::Confirmed,
2191 confidence: 1.0,
2192 warnings: Vec::new(),
2193 });
2194 }
2195}
2196
2197fn python_imported_symbol_paths(tokens: &[Token]) -> BTreeMap<String, String> {
2198 let mut output = BTreeMap::new();
2199 for index in 0..tokens.len() {
2200 if !tokens[index].is_ident("from") {
2201 continue;
2202 }
2203 let import_index = (index + 1..tokens.len())
2204 .take_while(|candidate| tokens[*candidate].line == tokens[index].line)
2205 .find(|candidate| tokens[*candidate].is_ident("import"));
2206 let Some(import_index) = import_index else {
2207 continue;
2208 };
2209 let module = tokens[index + 1..import_index]
2210 .iter()
2211 .filter_map(Token::ident)
2212 .collect::<Vec<_>>()
2213 .join("/");
2214 if module.is_empty() {
2215 continue;
2216 }
2217 let target_path = format!("{module}.py");
2218 let mut cursor = import_index + 1;
2219 while cursor < tokens.len() && tokens[cursor].line == tokens[index].line {
2220 let Some(imported) = tokens[cursor].ident() else {
2221 cursor += 1;
2222 continue;
2223 };
2224 let (local, advance) = if tokens
2225 .get(cursor + 1)
2226 .is_some_and(|token| token.is_ident("as"))
2227 {
2228 (
2229 tokens
2230 .get(cursor + 2)
2231 .and_then(Token::ident)
2232 .unwrap_or(imported),
2233 3,
2234 )
2235 } else {
2236 (imported, 1)
2237 };
2238 output.insert(local.to_owned(), target_path.clone());
2239 cursor += advance;
2240 }
2241 }
2242 output
2243}
2244
2245fn parse_python_http_calls(
2246 tokens: &[Token],
2247 functions: &[FunctionSpan],
2248 contexts: &PythonContexts,
2249 observations: &mut SourceObservationCollector<'_>,
2250) {
2251 let receivers = contexts
2252 .request_modules
2253 .iter()
2254 .chain(&contexts.request_clients)
2255 .map(|name| (name.as_str(), SourceFramework::Requests))
2256 .chain(
2257 contexts
2258 .httpx_modules
2259 .iter()
2260 .chain(&contexts.httpx_clients)
2261 .map(|name| (name.as_str(), SourceFramework::Httpx)),
2262 )
2263 .chain(
2264 contexts
2265 .aiohttp_clients
2266 .iter()
2267 .map(|name| (name.as_str(), SourceFramework::AioHttp)),
2268 )
2269 .collect::<BTreeMap<_, _>>();
2270 for index in 0..tokens.len() {
2271 let Some(receiver) = tokens[index].ident() else {
2272 continue;
2273 };
2274 let (framework, call, open) = if let Some(framework) = receivers.get(receiver).copied() {
2275 if !tokens
2276 .get(index + 1)
2277 .is_some_and(|token| token.is_punct('.'))
2278 {
2279 continue;
2280 }
2281 let Some(call) = tokens.get(index + 2).and_then(Token::ident) else {
2282 continue;
2283 };
2284 (framework, call, index + 3)
2285 } else if let Some((framework, call)) = contexts.direct_calls.get(receiver) {
2286 (*framework, call.as_str(), index + 1)
2287 } else {
2288 continue;
2289 };
2290 if !tokens.get(open).is_some_and(|token| token.is_punct('(')) {
2291 continue;
2292 }
2293 let Some(close) = matching(tokens, open, '(', ')') else {
2294 continue;
2295 };
2296 let arguments = top_level_arguments(tokens, open, close);
2297 let (method, path_argument) = if call == "request" {
2298 (
2299 arguments
2300 .first()
2301 .and_then(|argument| tokens.get(*argument))
2302 .and_then(Token::literal)
2303 .and_then(canonical_method)
2304 .map(str::to_owned),
2305 arguments.get(1).copied(),
2306 )
2307 } else {
2308 (
2309 canonical_method(call).map(str::to_owned),
2310 arguments.first().copied(),
2311 )
2312 };
2313 if method.is_none() && call != "request" {
2314 continue;
2315 }
2316 let resolved_path = path_argument.and_then(|argument| {
2317 python_static_string_expression(
2318 tokens,
2319 argument,
2320 python_argument_end(tokens, argument, close),
2321 &contexts.string_constants,
2322 )
2323 });
2324 let literal = resolved_path.as_deref().or_else(|| {
2325 path_argument
2326 .and_then(|argument| tokens.get(argument))
2327 .and_then(Token::literal)
2328 });
2329 let mut observation = http_from_literal(
2330 SourceLanguage::Python,
2331 framework,
2332 SourceRole::Consumer,
2333 method,
2334 literal,
2335 enclosing_symbol(functions, index),
2336 SourceLineRange {
2337 start: tokens[index].line,
2338 end: tokens[close].end_line,
2339 },
2340 false,
2341 );
2342 if call == "request" && observation.method.is_none() {
2343 observation.status = SourceEpistemicStatus::Incomplete;
2344 observation.confidence = 0.0;
2345 observation.warnings.push(SourceWarning::DynamicMethod);
2346 observation.warnings.sort();
2347 observation.warnings.dedup();
2348 }
2349 observations.push(observation);
2350 }
2351}
2352
2353fn parse_python_tests(
2354 source: &str,
2355 tokens: &[Token],
2356 observations: &mut SourceObservationCollector<'_>,
2357) {
2358 let unittest_classes = python_unittest_classes(source, tokens);
2359 let python_classes = python_class_ranges(source, tokens);
2360 for index in 0..tokens.len() {
2361 if !tokens[index].is_ident("def") {
2362 continue;
2363 }
2364 let Some(name) = tokens.get(index + 1).and_then(Token::ident) else {
2365 continue;
2366 };
2367 if !name.starts_with("test_") {
2368 continue;
2369 }
2370 let signature_end = (index + 2..tokens.len())
2371 .find(|candidate| tokens[*candidate].is_punct(':'))
2372 .unwrap_or(index + 1);
2373 let line = tokens[index].line;
2374 let is_unittest = unittest_classes
2375 .iter()
2376 .any(|range| range.start <= line && line <= range.end);
2377 let indent = source_indent(source, line);
2378 let is_class_method = python_classes.iter().any(|(range, body_indent)| {
2379 range.start <= line && line <= range.end && indent == *body_indent
2380 });
2381 if !is_unittest && indent != 0 && !is_class_method {
2382 continue;
2383 }
2384 observations.push(confirmed_test(
2385 SourceLanguage::Python,
2386 if is_unittest {
2387 SourceFramework::Unittest
2388 } else if is_class_method {
2389 SourceFramework::PythonTest
2390 } else {
2391 SourceFramework::Pytest
2392 },
2393 name.to_owned(),
2394 line,
2395 tokens[signature_end].end_line,
2396 ));
2397 }
2398}
2399
2400fn python_class_ranges(source: &str, tokens: &[Token]) -> Vec<(SourceLineRange, usize)> {
2401 let fallback = saturating_u32(source.lines().count().max(1));
2402 let mut classes = Vec::new();
2403 for token in tokens.iter().filter(|token| token.is_ident("class")) {
2404 let class_indent = source_indent(source, token.line);
2405 let end = python_block_end(source, token.line, class_indent, fallback);
2406 let body_indent = source
2407 .lines()
2408 .enumerate()
2409 .skip(usize::try_from(token.line).unwrap_or(usize::MAX))
2410 .take(usize::try_from(end.saturating_sub(token.line)).unwrap_or(usize::MAX))
2411 .find_map(|(index, line)| {
2412 let trimmed = line.trim();
2413 let line_number = saturating_u32(index + 1);
2414 let indent = source_indent(source, line_number);
2415 (!trimmed.is_empty() && !trimmed.starts_with('#') && indent > class_indent)
2416 .then_some(indent)
2417 });
2418 if let Some(body_indent) = body_indent {
2419 classes.push((
2420 SourceLineRange {
2421 start: token.line,
2422 end,
2423 },
2424 body_indent,
2425 ));
2426 }
2427 }
2428 classes
2429}
2430
2431fn python_unittest_classes(source: &str, tokens: &[Token]) -> Vec<SourceLineRange> {
2432 if !has_ident(tokens, "unittest") {
2433 return Vec::new();
2434 }
2435 let fallback = saturating_u32(source.lines().count().max(1));
2436 let mut classes = Vec::new();
2437 for index in 0..tokens.len() {
2438 if !tokens[index].is_ident("class") {
2439 continue;
2440 }
2441 let signature_end = (index + 1..tokens.len())
2442 .find(|candidate| tokens[*candidate].is_punct(':'))
2443 .unwrap_or(index);
2444 if !contains_ident(tokens, index + 1, signature_end, "TestCase") {
2445 continue;
2446 }
2447 let start = tokens[index].line;
2448 classes.push(SourceLineRange {
2449 start,
2450 end: python_block_end(source, start, source_indent(source, start), fallback),
2451 });
2452 }
2453 classes
2454}
2455
2456fn saturating_u32(value: usize) -> u32 {
2457 u32::try_from(value).unwrap_or(u32::MAX)
2458}
2459
2460fn finish(mut observations: Vec<SourceObservation>) -> Vec<SourceObservation> {
2461 for observation in &mut observations {
2462 observation.warnings.sort();
2463 observation.warnings.dedup();
2464 }
2465 observations.sort_by(compare_observations);
2466 observations.dedup();
2467 observations
2468}
2469
2470fn compare_observations(left: &SourceObservation, right: &SourceObservation) -> Ordering {
2471 (
2472 left.lines,
2473 left.language,
2474 left.framework,
2475 left.role,
2476 &left.method,
2477 &left.path,
2478 &left.symbol_name,
2479 &left.related_symbol,
2480 &left.related_path,
2481 left.status,
2482 &left.warnings,
2483 )
2484 .cmp(&(
2485 right.lines,
2486 right.language,
2487 right.framework,
2488 right.role,
2489 &right.method,
2490 &right.path,
2491 &right.symbol_name,
2492 &right.related_symbol,
2493 &right.related_path,
2494 right.status,
2495 &right.warnings,
2496 ))
2497}
2498
2499#[cfg(test)]
2500mod tests {
2501 use super::{
2502 SourceEpistemicStatus, SourceFramework, SourceRole, SourceWarning, parse_python_source, parse_rust_source
2503 };
2504
2505 #[test]
2506 fn axum_should_extract_multiline_route_and_handler() {
2507 let source = r#"
2508use axum::{Router, routing::post};
2509fn router() {
2510 Router::new().route(
2511 "/api//orders/",
2512 post(create_order),
2513 );
2514}
2515"#;
2516 let result = parse_rust_source(source);
2517
2518 assert!(matches!(
2519 result.as_slice(),
2520 [item]
2521 if item.framework == SourceFramework::Axum
2522 && item.role == SourceRole::Provider
2523 && item.method.as_deref() == Some("POST")
2524 && item.path.as_deref() == Some("/api/orders")
2525 && item.symbol_name.as_deref() == Some("create_order")
2526 && item.lines.start == 4
2527 && item.lines.end == 7
2528 ));
2529 }
2530
2531 #[test]
2532 fn actix_attributes_should_extract_direct_and_route_methods() {
2533 let source = r#"
2534use actix_web::{get, route};
2535#[get("/health")]
2536async fn health() {}
2537#[route(
2538 "/orders",
2539 method = "POST",
2540)]
2541async fn create() {}
2542"#;
2543 let result = parse_rust_source(source);
2544
2545 assert_eq!(
2546 result
2547 .iter()
2548 .map(|item| (item.method.as_deref(), item.path.as_deref()))
2549 .collect::<Vec<_>>(),
2550 vec![
2551 (Some("GET"), Some("/health")),
2552 (Some("POST"), Some("/orders"))
2553 ]
2554 );
2555 }
2556
2557 #[test]
2558 fn actix_builder_should_require_web_route_evidence() {
2559 let source = r#"
2560use actix_web::{web, App};
2561fn app() {
2562 App::new().route("/users", web::put().to(update_user));
2563 App::new().service(
2564 web::resource("/orders").route(web::post().to(create_order)),
2565 );
2566}
2567"#;
2568 let result = parse_rust_source(source);
2569
2570 assert_eq!(
2571 result
2572 .iter()
2573 .map(|item| (
2574 item.framework,
2575 item.method.as_deref(),
2576 item.symbol_name.as_deref()
2577 ))
2578 .collect::<Vec<_>>(),
2579 vec![
2580 (SourceFramework::ActixWeb, Some("PUT"), Some("update_user")),
2581 (
2582 SourceFramework::ActixWeb,
2583 Some("POST"),
2584 Some("create_order")
2585 ),
2586 ]
2587 );
2588 }
2589
2590 #[test]
2591 fn utoipa_should_extract_explicit_and_contextual_actix_paths() {
2592 let source = r#"
2593use actix_web::{get, post};
2594
2595#[cfg_attr(feature = "openapi", utoipa::path(post, path = "/api/orders"))]
2596#[post("/orders")]
2597async fn create_order() {}
2598
2599#[cfg_attr(feature = "openapi", utoipa::path(
2600 get,
2601 context_path = "/api/users"
2602))]
2603#[get("/{id}")]
2604async fn get_user() {}
2605"#;
2606 let result = parse_rust_source(source);
2607
2608 assert!(result.iter().any(|item| {
2609 item.framework == SourceFramework::Utoipa
2610 && item.symbol_name.as_deref() == Some("create_order")
2611 && item.method.as_deref() == Some("POST")
2612 && item.path.as_deref() == Some("/api/orders")
2613 }));
2614 assert!(result.iter().any(|item| {
2615 item.framework == SourceFramework::Utoipa
2616 && item.symbol_name.as_deref() == Some("get_user")
2617 && item.method.as_deref() == Some("GET")
2618 && item.path.as_deref() == Some("/api/users/{id}")
2619 }));
2620 assert!(result.iter().any(|item| {
2621 item.framework == SourceFramework::ActixWeb
2622 && item.symbol_name.as_deref() == Some("create_order")
2623 && item.path.as_deref() == Some("/orders")
2624 && (item.confidence - 1.0).abs() < f32::EPSILON
2625 }));
2626 assert!(result.iter().any(|item| {
2627 item.framework == SourceFramework::Utoipa
2628 && item.symbol_name.as_deref() == Some("create_order")
2629 && (item.confidence - 0.75).abs() < f32::EPSILON
2630 && item.warnings.contains(&SourceWarning::AdvisoryDeclaration)
2631 }));
2632 }
2633
2634 #[test]
2635 fn reqwest_should_extract_convenience_client_and_request_calls() {
2636 let source = r#"
2637use reqwest::{Client, Method};
2638async fn send() {
2639 reqwest::get("https://example.test/health?full=1").await;
2640 reqwest::blocking::get("https://example.test/ready");
2641 let client = Client::new();
2642 client.post("/orders").send().await;
2643 client.request(Method::DELETE, "https://example.test/orders/7").send().await;
2644}
2645"#;
2646 let result = parse_rust_source(source);
2647
2648 assert_eq!(
2649 result
2650 .iter()
2651 .map(|item| (item.method.as_deref(), item.path.as_deref()))
2652 .collect::<Vec<_>>(),
2653 vec![
2654 (Some("GET"), Some("/health")),
2655 (Some("GET"), Some("/ready")),
2656 (Some("POST"), Some("/orders")),
2657 (Some("DELETE"), Some("/orders/7")),
2658 ]
2659 );
2660 }
2661
2662 #[test]
2663 fn rust_test_attributes_should_cover_builtin_tokio_and_rstest() {
2664 let source = r"
2665#[test]
2666fn plain() {}
2667#[tokio::test]
2668async fn asynchronous() {}
2669#[rstest]
2670#[case(1)]
2671fn parameterized(#[case] value: u8) {}
2672";
2673 let result = parse_rust_source(source);
2674
2675 assert_eq!(
2676 result
2677 .iter()
2678 .map(|item| (item.framework, item.symbol_name.as_deref()))
2679 .collect::<Vec<_>>(),
2680 vec![
2681 (SourceFramework::RustTest, Some("plain")),
2682 (SourceFramework::TokioTest, Some("asynchronous")),
2683 (SourceFramework::Rstest, Some("parameterized")),
2684 ]
2685 );
2686 }
2687
2688 #[test]
2689 fn rust_dynamic_route_should_never_claim_exact_path() {
2690 let source = r"
2691use axum::{Router, routing::get};
2692fn router(path: &str) {
2693 Router::new().route(path, get(handler));
2694}
2695";
2696 let result = parse_rust_source(source);
2697
2698 assert!(matches!(
2699 result.as_slice(),
2700 [item]
2701 if item.path.is_none()
2702 && item.status == SourceEpistemicStatus::Ambiguous
2703 && item.warnings == [SourceWarning::DynamicPath]
2704 ));
2705 }
2706
2707 #[test]
2708 fn rust_comments_strings_and_unrelated_route_methods_should_be_ignored() {
2709 let source = r##"
2710// use axum; Router::new().route("/fake", get(fake));
2711const TEXT: &str = r#"reqwest::get("https://example.test/fake")"#;
2712struct Router;
2713impl Router {
2714 fn route(&self, path: &str, handler: usize) {}
2715}
2716fn ordinary() {
2717 Router.route("/fake", handler);
2718}
2719"##;
2720 let result = parse_rust_source(source);
2721
2722 assert!(result.is_empty());
2723 }
2724
2725 #[test]
2726 fn fastapi_should_extract_method_and_api_route_decorators() {
2727 let source = r#"
2728from fastapi import FastAPI
2729app = FastAPI()
2730@app.get("/users/{user_id}")
2731async def get_user(user_id: str):
2732 pass
2733@app.api_route(
2734 "/orders",
2735 methods=["POST", "PUT"],
2736)
2737def orders():
2738 pass
2739"#;
2740 let result = parse_python_source(source);
2741
2742 assert_eq!(
2743 result
2744 .iter()
2745 .filter(|item| item.role == SourceRole::Provider)
2746 .map(|item| (item.method.as_deref(), item.path.as_deref()))
2747 .collect::<Vec<_>>(),
2748 vec![
2749 (Some("GET"), Some("/users/{user_id}")),
2750 (Some("POST"), Some("/orders")),
2751 (Some("PUT"), Some("/orders")),
2752 ]
2753 );
2754 }
2755
2756 #[test]
2757 fn fastapi_should_compose_static_router_prefixes() {
2758 let source = r#"
2759from fastapi import APIRouter
2760router = APIRouter(prefix="/api/v1/projects")
2761
2762@router.get("/{project_id}")
2763async def get_project(project_id: int):
2764 pass
2765"#;
2766 let result = parse_python_source(source);
2767
2768 assert!(result.iter().any(|item| {
2769 item.framework == SourceFramework::FastApi
2770 && item.method.as_deref() == Some("GET")
2771 && item.path.as_deref() == Some("/api/v1/projects/{project_id}")
2772 && item.symbol_name.as_deref() == Some("get_project")
2773 }));
2774 }
2775
2776 #[test]
2777 fn flask_should_extract_default_and_explicit_methods() {
2778 let source = r#"
2779from flask import Flask
2780app = Flask(__name__)
2781@app.route("/health")
2782def health():
2783 pass
2784@app.route(
2785 "/orders",
2786 methods=["POST", "DELETE"],
2787)
2788def orders():
2789 pass
2790"#;
2791 let result = parse_python_source(source);
2792
2793 assert_eq!(
2794 result
2795 .iter()
2796 .filter(|item| item.role == SourceRole::Provider)
2797 .map(|item| item.method.as_deref())
2798 .collect::<Vec<_>>(),
2799 vec![Some("GET"), Some("DELETE"), Some("POST")]
2800 );
2801 }
2802
2803 #[test]
2804 fn flask_should_resolve_instance_app_and_static_path_concatenation() {
2805 let source = r#"
2806from flask import Flask
2807URL = "/servers/remitee"
2808
2809class Remitee:
2810 def __init__(self):
2811 self.app = Flask(__name__)
2812
2813 def setup_routes(self):
2814 @self.app.route(URL + "/api/Payments/<payment_id>", methods=["GET"])
2815 def payment(payment_id):
2816 pass
2817"#;
2818 let result = parse_python_source(source);
2819
2820 assert!(result.iter().any(|item| {
2821 item.framework == SourceFramework::Flask
2822 && item.role == SourceRole::Provider
2823 && item.method.as_deref() == Some("GET")
2824 && item.path.as_deref() == Some("/servers/remitee/api/Payments/<payment_id>")
2825 }));
2826 }
2827
2828 #[test]
2829 fn python_http_registry_should_flatten_nested_static_operations() {
2830 let source = r#"
2831METHODS = {
2832 "public": ("/api/v1", {
2833 "accounts": ("/accounts", {
2834 "get_accounts": ["GET", ""],
2835 "get_account": ["GET", "/{{accounts_id}}"],
2836 }),
2837 }),
2838}
2839"#;
2840 let result = parse_python_source(source);
2841 let operations = result
2842 .iter()
2843 .filter(|item| item.framework == SourceFramework::PythonHttpRegistry)
2844 .map(|item| (item.method.as_deref(), item.path.as_deref()))
2845 .collect::<Vec<_>>();
2846
2847 assert_eq!(
2848 operations,
2849 vec![
2850 (Some("GET"), Some("/api/v1/accounts")),
2851 (Some("GET"), Some("/api/v1/accounts/{accounts_id}")),
2852 ]
2853 );
2854 }
2855
2856 #[test]
2857 fn factory_boy_should_link_factory_to_imported_model() {
2858 let source = r"
2859import factory
2860from src.clases.Account import Account
2861
2862class AccountFactory(factory.Factory):
2863 class Meta:
2864 model = Account
2865";
2866 let result = parse_python_source(source);
2867
2868 assert!(result.iter().any(|item| {
2869 item.framework == SourceFramework::FactoryBoy
2870 && item.role == SourceRole::Factory
2871 && item.symbol_name.as_deref() == Some("AccountFactory")
2872 && item.related_symbol.as_deref() == Some("Account")
2873 && item.related_path.as_deref() == Some("src/clases/Account.py")
2874 && item.status == SourceEpistemicStatus::Confirmed
2875 }));
2876 }
2877
2878 #[test]
2879 fn requests_should_extract_module_alias_and_session_calls() {
2880 let source = r#"
2881import requests as rq
2882from requests import Session, post as create
2883def send():
2884 rq.get("https://example.test/status")
2885 create("/orders")
2886 session = Session()
2887 session.request("PATCH", "/orders/4")
2888"#;
2889 let result = parse_python_source(source);
2890
2891 assert_eq!(
2892 result
2893 .iter()
2894 .map(|item| (item.framework, item.method.as_deref(), item.path.as_deref()))
2895 .collect::<Vec<_>>(),
2896 vec![
2897 (SourceFramework::Requests, Some("GET"), Some("/status")),
2898 (SourceFramework::Requests, Some("POST"), Some("/orders")),
2899 (SourceFramework::Requests, Some("PATCH"), Some("/orders/4")),
2900 ]
2901 );
2902 }
2903
2904 #[test]
2905 fn aiohttp_should_extract_client_session_context_calls() {
2906 let source = r#"
2907from aiohttp import ClientSession
2908
2909async def load():
2910 async with ClientSession() as session:
2911 await session.get("/accounts")
2912"#;
2913 let result = parse_python_source(source);
2914
2915 assert!(result.iter().any(|item| {
2916 item.framework == SourceFramework::AioHttp
2917 && item.role == SourceRole::Consumer
2918 && item.method.as_deref() == Some("GET")
2919 && item.path.as_deref() == Some("/accounts")
2920 }));
2921 }
2922
2923 #[test]
2924 fn httpx_should_extract_module_sync_and_async_clients() {
2925 let source = r#"
2926import httpx
2927from httpx import AsyncClient, patch as update
2928def direct():
2929 httpx.delete("https://example.test/items/1")
2930 update("/items/1")
2931async def clients():
2932 client = AsyncClient()
2933 client.put("/items/1")
2934"#;
2935 let result = parse_python_source(source);
2936
2937 assert_eq!(
2938 result
2939 .iter()
2940 .map(|item| (item.framework, item.method.as_deref()))
2941 .collect::<Vec<_>>(),
2942 vec![
2943 (SourceFramework::Httpx, Some("DELETE")),
2944 (SourceFramework::Httpx, Some("PATCH")),
2945 (SourceFramework::Httpx, Some("PUT")),
2946 ]
2947 );
2948 }
2949
2950 #[test]
2951 fn python_tests_should_distinguish_pytest_and_unittest() {
2952 let source = r"
2953import unittest
2954def test_pytest_case():
2955 pass
2956class ApiTests(unittest.TestCase):
2957 def test_unittest_case(self):
2958 pass
2959";
2960 let result = parse_python_source(source);
2961
2962 assert_eq!(
2963 result
2964 .iter()
2965 .map(|item| (item.framework, item.symbol_name.as_deref()))
2966 .collect::<Vec<_>>(),
2967 vec![
2968 (SourceFramework::Pytest, Some("test_pytest_case")),
2969 (SourceFramework::Unittest, Some("test_unittest_case")),
2970 ]
2971 );
2972 }
2973
2974 #[test]
2975 fn indirect_python_test_class_should_preserve_generic_test_evidence() {
2976 let source = r#"
2977import requests
2978
2979class PaymentCases(SharedTestBase):
2980 def test_payment(self):
2981 requests.post("/payments")
2982"#;
2983 let result = parse_python_source(source);
2984
2985 assert!(result.iter().any(|item| {
2986 item.framework == SourceFramework::PythonTest
2987 && item.role == SourceRole::Test
2988 && item.symbol_name.as_deref() == Some("test_payment")
2989 }));
2990 assert!(result.iter().any(|item| {
2991 item.framework == SourceFramework::Requests
2992 && item.role == SourceRole::Consumer
2993 && item.symbol_name.as_deref() == Some("test_payment")
2994 && item.path.as_deref() == Some("/payments")
2995 }));
2996 }
2997
2998 #[test]
2999 fn python_dynamic_url_and_method_should_remain_explicitly_incomplete() {
3000 let source = r"
3001import httpx
3002def send(method, base, path):
3003 httpx.request(method, base + path)
3004";
3005 let result = parse_python_source(source);
3006
3007 assert!(matches!(
3008 result.as_slice(),
3009 [item]
3010 if item.method.is_none()
3011 && item.path.is_none()
3012 && item.status == SourceEpistemicStatus::Incomplete
3013 && item.warnings
3014 == [SourceWarning::DynamicMethod, SourceWarning::DynamicPath]
3015 ));
3016 }
3017
3018 #[test]
3019 fn python_f_strings_should_be_dynamic_not_exact() {
3020 let source = r#"
3021import requests
3022def fetch(user_id):
3023 requests.get(f"https://example.test/users/{user_id}")
3024"#;
3025 let result = parse_python_source(source);
3026
3027 assert!(matches!(
3028 result.as_slice(),
3029 [item]
3030 if item.path.is_none()
3031 && item.status == SourceEpistemicStatus::Ambiguous
3032 && item.warnings == [SourceWarning::DynamicPath]
3033 ));
3034 }
3035
3036 #[test]
3037 fn python_comments_docstrings_and_unknown_clients_should_be_ignored() {
3038 let source = r#"
3039"""@app.get("/fake")
3040def test_fake(): pass
3041requests.get("https://example.test/fake")
3042"""
3043# import requests
3044# requests.post("/fake")
3045class Client:
3046 def get(self, path):
3047 pass
3048client = Client()
3049client.get("/not-httpx")
3050"#;
3051 let result = parse_python_source(source);
3052
3053 assert!(result.is_empty());
3054 }
3055
3056 #[test]
3057 fn ordering_should_be_source_stable_and_independent_of_method_list_order() {
3058 let source = r#"
3059from fastapi import FastAPI
3060app = FastAPI()
3061@app.api_route("/z", methods=["PUT", "GET", "POST"])
3062def endpoint():
3063 pass
3064"#;
3065 let first = parse_python_source(source);
3066 let second = parse_python_source(source);
3067
3068 assert_eq!(
3069 (
3070 first
3071 .iter()
3072 .map(|item| item.method.as_deref())
3073 .collect::<Vec<_>>(),
3074 &first
3075 ),
3076 ((vec![Some("GET"), Some("POST"), Some("PUT")]), &second)
3077 );
3078 }
3079}