Skip to main content

threatflux_package_security/analyzers/
python.rs

1//! Python package analyzer
2
3use anyhow::Result;
4use async_trait::async_trait;
5use serde::{Deserialize, Serialize};
6use std::collections::{BTreeMap, BTreeSet};
7use std::path::Path;
8
9use crate::core::{
10    AnalysisResult, Dependency, DependencyAnalysis, DependencyType, MaliciousPattern,
11    PackageAnalyzer, PackageInfo, PackageMetadata, PatternMatcher, RiskAssessment, RiskCalculator,
12    SupplyChainSignal, Vulnerability,
13};
14use crate::utils::typosquatting::{PackageEcosystem, TyposquattingDetector};
15use crate::vulnerability_db::VulnerabilityDatabase;
16
17/// Python package information
18#[derive(Debug, Clone, Serialize, Deserialize)]
19pub struct PythonPackage {
20    pub metadata: PackageMetadata,
21}
22
23impl PackageInfo for PythonPackage {
24    fn metadata(&self) -> &PackageMetadata {
25        &self.metadata
26    }
27
28    fn package_type(&self) -> &str {
29        "python"
30    }
31
32    fn custom_attributes(&self) -> BTreeMap<String, serde_json::Value> {
33        BTreeMap::new()
34    }
35}
36
37/// Python analysis result
38#[derive(Debug, Clone, Serialize, Deserialize)]
39pub struct PythonAnalysisResult {
40    pub package: PythonPackage,
41    pub risk_assessment: RiskAssessment,
42    pub dependency_analysis: DependencyAnalysis,
43    /// Advisory matches for the package being analyzed, excluding its
44    /// dependencies.
45    pub subject_vulnerabilities: Vec<Vulnerability>,
46    /// Deterministic union of subject and dependency advisory matches.
47    pub vulnerabilities: Vec<Vulnerability>,
48    pub malicious_patterns: Vec<MaliciousPattern>,
49    pub setup_analysis: SetupAnalysis,
50    pub typosquatting_risk: Option<TyposquattingRisk>,
51    pub vulnerability_database: crate::core::DatabaseMetadata,
52}
53
54impl AnalysisResult for PythonAnalysisResult {
55    fn package_info(&self) -> &dyn PackageInfo {
56        &self.package
57    }
58
59    fn risk_assessment(&self) -> &RiskAssessment {
60        &self.risk_assessment
61    }
62
63    fn dependency_analysis(&self) -> &DependencyAnalysis {
64        &self.dependency_analysis
65    }
66
67    fn vulnerabilities(&self) -> &[Vulnerability] {
68        &self.vulnerabilities
69    }
70
71    fn subject_vulnerabilities(&self) -> &[Vulnerability] {
72        &self.subject_vulnerabilities
73    }
74
75    fn malicious_patterns(&self) -> &[MaliciousPattern] {
76        &self.malicious_patterns
77    }
78
79    fn vulnerability_database_metadata(&self) -> Option<&crate::core::DatabaseMetadata> {
80        Some(&self.vulnerability_database)
81    }
82
83    fn to_json(&self) -> Result<serde_json::Value> {
84        Ok(serde_json::to_value(self)?)
85    }
86
87    fn typosquatting_risk(&self) -> Option<crate::core::TyposquattingRisk> {
88        self.typosquatting_risk
89            .as_ref()
90            .map(|risk| crate::core::TyposquattingRisk {
91                is_potential_typosquatting: risk.is_likely_typosquatting,
92                similar_packages: risk.similar_packages.clone(),
93                confidence_score: risk.confidence,
94            })
95    }
96}
97
98/// Python setup.py analysis
99#[derive(Debug, Clone, Serialize, Deserialize)]
100pub struct SetupAnalysis {
101    pub has_setup_py: bool,
102    pub has_custom_commands: bool,
103    pub dangerous_operations: Vec<String>,
104    pub external_downloads: Vec<String>,
105    pub code_execution_risk: bool,
106}
107
108#[derive(Debug, Clone, Serialize, Deserialize)]
109pub struct TyposquattingRisk {
110    pub is_likely_typosquatting: bool,
111    pub similar_packages: Vec<String>,
112    pub confidence: f32,
113}
114
115/// Python package analyzer
116pub struct PythonAnalyzer {
117    vuln_db: Box<dyn VulnerabilityDatabase>,
118    pattern_matcher: PatternMatcher,
119    typo_detector: TyposquattingDetector,
120}
121
122impl PythonAnalyzer {
123    /// Create a new Python analyzer
124    pub fn new() -> Result<Self> {
125        Ok(Self {
126            vuln_db: crate::vulnerability_db::create_python_database(),
127            pattern_matcher: PatternMatcher::new()?,
128            typo_detector: TyposquattingDetector::for_ecosystem(PackageEcosystem::Python),
129        })
130    }
131
132    /// Create an analyzer using an application-provided vulnerability database.
133    pub fn with_database(vulnerability_database: Box<dyn VulnerabilityDatabase>) -> Result<Self> {
134        Ok(Self {
135            vuln_db: vulnerability_database,
136            pattern_matcher: PatternMatcher::new()?,
137            typo_detector: TyposquattingDetector::for_ecosystem(PackageEcosystem::Python),
138        })
139    }
140
141    /// Parse setup.py or pyproject.toml
142    async fn parse_package_metadata(&self, path: &Path) -> Result<PythonPackage> {
143        if !path.is_dir() {
144            anyhow::bail!("Python analysis currently supports project directories only");
145        }
146
147        let metadata = if std::fs::symlink_metadata(path.join("pyproject.toml")).is_ok() {
148            let content = crate::utils::input::read_project_file(
149                path,
150                Path::new("pyproject.toml"),
151                crate::limits::MAX_PROJECT_FILE_BYTES,
152            )
153            .await?;
154            let document = parse_toml_document(&content, "pyproject.toml")?;
155            if document.get("project").is_some() {
156                self.parse_pyproject_document(&document)?
157            } else {
158                self.parse_legacy_metadata(path).await?
159            }
160        } else {
161            self.parse_legacy_metadata(path).await?
162        };
163
164        validate_python_metadata(&metadata)?;
165        Ok(PythonPackage { metadata })
166    }
167
168    async fn parse_legacy_metadata(&self, path: &Path) -> Result<PackageMetadata> {
169        if std::fs::symlink_metadata(path.join("setup.cfg")).is_ok() {
170            let content = crate::utils::input::read_project_file(
171                path,
172                Path::new("setup.cfg"),
173                crate::limits::MAX_PROJECT_FILE_BYTES,
174            )
175            .await?;
176            self.parse_setup_cfg(&content)
177        } else if std::fs::symlink_metadata(path.join("setup.py")).is_ok() {
178            let content = crate::utils::input::read_project_file(
179                path,
180                Path::new("setup.py"),
181                crate::limits::MAX_PROJECT_FILE_BYTES,
182            )
183            .await?;
184            self.parse_setup_py(&content)
185        } else {
186            anyhow::bail!("No supported Python project metadata found")
187        }
188    }
189
190    /// Parse setup.py file
191    fn parse_setup_py(&self, content: &str) -> Result<PackageMetadata> {
192        let fields = extract_setup_literal_fields(content)?;
193        let required = |field: &str| {
194            fields
195                .get(field)
196                .cloned()
197                .ok_or_else(|| anyhow::anyhow!("setup() has no static string {field} field"))
198        };
199
200        Ok(PackageMetadata {
201            name: required("name")?,
202            version: required("version")?,
203            description: fields.get("description").cloned(),
204            author: fields.get("author").cloned(),
205            license: fields.get("license").cloned(),
206            homepage: fields.get("url").cloned(),
207            repository: None,
208            // setup.py list expressions are not evaluated or executed.
209            keywords: vec![],
210            publish_date: None,
211        })
212    }
213
214    fn parse_pyproject_document(&self, toml_value: &toml::Value) -> Result<PackageMetadata> {
215        let project = toml_value
216            .get("project")
217            .and_then(toml::Value::as_table)
218            .ok_or_else(|| anyhow::anyhow!("No [project] table in pyproject.toml"))?;
219
220        Ok(PackageMetadata {
221            name: project
222                .get("name")
223                .and_then(|v| v.as_str())
224                .ok_or_else(|| anyhow::anyhow!("Missing project name"))?
225                .to_string(),
226            version: project
227                .get("version")
228                .and_then(|v| v.as_str())
229                .ok_or_else(|| anyhow::anyhow!("Missing project version"))?
230                .to_string(),
231            description: optional_toml_string(project, "description")?,
232            author: parse_pyproject_authors(project.get("authors"))?,
233            license: parse_pyproject_license(project.get("license"))?,
234            homepage: parse_project_url(project, "Homepage")?,
235            repository: parse_project_url(project, "Repository")?,
236            keywords: parse_toml_string_array(project.get("keywords"), "project.keywords")?,
237            publish_date: None,
238        })
239    }
240
241    /// Parse setup.cfg file
242    fn parse_setup_cfg(&self, content: &str) -> Result<PackageMetadata> {
243        // Simple INI-style parsing
244        let mut metadata = PackageMetadata {
245            name: String::new(),
246            version: String::new(),
247            description: None,
248            author: None,
249            license: None,
250            homepage: None,
251            repository: None,
252            keywords: vec![],
253            publish_date: None,
254        };
255
256        let mut in_metadata_section = false;
257        let mut saw_metadata_section = false;
258        let mut metadata_keys = BTreeSet::new();
259
260        for line in content.lines() {
261            let trimmed = line.trim();
262            if trimmed.eq_ignore_ascii_case("[metadata]") {
263                if saw_metadata_section {
264                    anyhow::bail!("setup.cfg contains duplicate metadata sections");
265                }
266                saw_metadata_section = true;
267                in_metadata_section = true;
268                continue;
269            }
270            if trimmed.starts_with('[') {
271                in_metadata_section = false;
272            }
273
274            if in_metadata_section && let Some((key, value)) = line.split_once('=') {
275                let key = key.trim().to_ascii_lowercase();
276                let value = value.trim();
277                if !metadata_keys.insert(key.clone()) {
278                    anyhow::bail!("setup.cfg contains a duplicate metadata option");
279                }
280
281                match key.as_str() {
282                    "name" => metadata.name = value.to_string(),
283                    "version" => metadata.version = value.to_string(),
284                    "description" => metadata.description = Some(value.to_string()),
285                    "author" => metadata.author = Some(value.to_string()),
286                    "license" => metadata.license = Some(value.to_string()),
287                    "url" => metadata.homepage = Some(value.to_string()),
288                    _ => {}
289                }
290            }
291        }
292
293        if metadata.name.is_empty() || metadata.version.is_empty() {
294            return Err(anyhow::anyhow!("Missing required metadata in setup.cfg"));
295        }
296
297        Ok(metadata)
298    }
299
300    /// Analyze setup.py for dangerous operations
301    fn analyze_setup(&self, content: &str) -> SetupAnalysis {
302        let mut analysis = SetupAnalysis {
303            has_setup_py: true,
304            has_custom_commands: false,
305            dangerous_operations: vec![],
306            external_downloads: vec![],
307            code_execution_risk: false,
308        };
309
310        // Check for custom commands
311        if contains_python_identifier(content, "cmdclass") {
312            analysis.has_custom_commands = true;
313            analysis
314                .dangerous_operations
315                .push("Custom setup commands detected".to_string());
316        }
317
318        // Check for dangerous operations
319        let dangerous_patterns = [
320            ("subprocess.run", "Process execution", true, true),
321            ("subprocess.call", "Process execution", true, true),
322            ("subprocess.check_call", "Process execution", true, true),
323            ("subprocess.check_output", "Process execution", true, true),
324            ("subprocess.Popen", "Process execution", true, true),
325            ("os.system", "System command execution", true, true),
326            ("exec", "Dynamic code execution", true, true),
327            ("eval", "Code evaluation", true, true),
328            ("__import__", "Dynamic imports", true, true),
329            ("urllib", "Network access", false, false),
330            ("requests", "HTTP requests", false, false),
331        ];
332
333        for (pattern, description, executes_code, requires_call) in &dangerous_patterns {
334            let matched = if *requires_call {
335                contains_python_function_call(content, pattern)
336            } else {
337                contains_python_identifier(content, pattern)
338            };
339            if matched {
340                analysis.dangerous_operations.push(description.to_string());
341                analysis.code_execution_risk |= executes_code;
342            }
343        }
344
345        // Check for external downloads
346        if contains_python_function_call(content, "urlopen")
347            || contains_python_function_call(content, "requests.get")
348        {
349            analysis
350                .external_downloads
351                .push("External download detected".to_string());
352        }
353
354        analysis
355    }
356
357    /// Analyze dependencies
358    async fn analyze_dependencies(&self, path: &Path) -> Result<DependencyAnalysis> {
359        let mut analysis = DependencyAnalysis::default();
360        let (requirements, source) = load_python_requirements(path).await?;
361        if requirements.len() > crate::limits::MAX_DIRECT_DEPENDENCIES {
362            anyhow::bail!("dependency count exceeds the configured limit");
363        }
364
365        let mut parsed_requirements = BTreeMap::new();
366        for raw_requirement in requirements {
367            let Some((name, version_spec, has_environment_marker)) =
368                parse_python_requirement(&raw_requirement)
369            else {
370                analysis.unresolved_requirements.push(raw_requirement);
371                continue;
372            };
373            if name.len() > crate::limits::MAX_METADATA_FIELD_BYTES
374                || version_spec.len() > crate::limits::MAX_METADATA_FIELD_BYTES
375            {
376                anyhow::bail!("Python requirement exceeds the configured field limit");
377            }
378            let is_exact =
379                crate::utils::version_parser::parse_exact_python_requirement(&version_spec).is_ok();
380            if has_environment_marker || !is_exact {
381                analysis
382                    .unresolved_requirements
383                    .push(raw_requirement.clone());
384            }
385
386            parsed_requirements
387                .entry((
388                    crate::utils::package_name::python(&name),
389                    version_spec.clone(),
390                    has_environment_marker,
391                ))
392                .or_insert((name, version_spec, has_environment_marker));
393        }
394
395        for (_, (name, version_spec, has_environment_marker)) in parsed_requirements {
396            // Without an application-supplied target environment, a marker's
397            // truth value is unknown. Do not strip it and turn a conditional
398            // declaration into an unconditional advisory match.
399            let vulns = if !has_environment_marker {
400                // Application-provided databases own their requirement
401                // semantics; the built-in snapshot independently enforces
402                // exact PEP 440 versions.
403                self.vuln_db
404                    .check_package(&name, &version_spec, "python")
405                    .await?
406            } else {
407                vec![]
408            };
409
410            let dependency = Dependency {
411                name,
412                version_spec,
413                resolved_version: None,
414                dependency_type: match source {
415                    PythonRequirementSource::Pep621 => DependencyType::Runtime,
416                    PythonRequirementSource::RequirementsFile => DependencyType::Unknown,
417                },
418                is_direct: match source {
419                    PythonRequirementSource::Pep621 => Some(true),
420                    PythonRequirementSource::RequirementsFile => None,
421                },
422                is_dev: match source {
423                    PythonRequirementSource::Pep621 => Some(false),
424                    PythonRequirementSource::RequirementsFile => None,
425                },
426                vulnerabilities: vulns,
427                license: None,
428                dependencies: vec![],
429            };
430
431            if dependency.is_direct == Some(true) {
432                analysis.direct_dependencies += 1;
433            }
434            analysis.dependency_tree.push(dependency);
435        }
436
437        analysis.total_dependencies = analysis.dependency_tree.len();
438        analysis.unresolved_requirements.sort();
439        analysis.unresolved_requirements.dedup();
440
441        analysis.rebuild_vulnerability_summary();
442
443        Ok(analysis)
444    }
445}
446
447#[async_trait]
448impl PackageAnalyzer for PythonAnalyzer {
449    type Package = PythonPackage;
450    type Analysis = PythonAnalysisResult;
451
452    async fn analyze(&self, path: &Path) -> Result<Self::Analysis> {
453        crate::utils::require_tokio_runtime()?;
454        let package = self.parse_package_metadata(path).await?;
455        let setup_content = if std::fs::symlink_metadata(path.join("setup.py")).is_ok() {
456            Some(
457                crate::utils::input::read_project_file(
458                    path,
459                    Path::new("setup.py"),
460                    crate::limits::MAX_PROJECT_FILE_BYTES,
461                )
462                .await?,
463            )
464        } else {
465            None
466        };
467        let dependency_analysis = self.analyze_dependencies(path).await?;
468        let subject_vulnerabilities = self
469            .vuln_db
470            .check_package(&package.metadata.name, &package.metadata.version, "python")
471            .await?;
472
473        let setup_code = setup_content
474            .as_deref()
475            .map(sanitize_python_source)
476            .unwrap_or_default();
477        let setup_analysis = if setup_content.is_some() {
478            self.analyze_setup(&setup_code)
479        } else {
480            SetupAnalysis {
481                has_setup_py: false,
482                has_custom_commands: false,
483                dangerous_operations: vec![],
484                external_downloads: vec![],
485                code_execution_risk: false,
486            }
487        };
488
489        // Surface heuristic text-pattern matches.
490        let malicious_patterns = self.pattern_matcher.scan(&setup_code, Some("setup.py"))?;
491
492        // Check typosquatting
493        let typosquatting_risk = if self.typo_detector.is_typosquatting(&package.metadata.name) {
494            Some(TyposquattingRisk {
495                is_likely_typosquatting: true,
496                similar_packages: self.typo_detector.find_similar(&package.metadata.name),
497                confidence: 0.8,
498            })
499        } else {
500            None
501        };
502
503        // Collect all vulnerabilities
504        let vulnerabilities = crate::core::dependency::deduplicate_vulnerabilities(
505            subject_vulnerabilities
506                .iter()
507                .cloned()
508                .chain(dependency_analysis.unique_vulnerabilities()),
509        );
510
511        // Calculate risk assessment
512        let risk_calculator = RiskCalculator::new();
513        let supply_chain_signal = if setup_analysis.code_execution_risk {
514            Some(SupplyChainSignal {
515                score: 50.0,
516                description: "Executable setup.py behavior detected".to_string(),
517                evidence: setup_analysis
518                    .dangerous_operations
519                    .iter()
520                    .take(crate::limits::MAX_EVIDENCE_PER_PATTERN)
521                    .cloned()
522                    .collect(),
523                mitigation: Some(
524                    "Review setup.py in an isolated environment before use".to_string(),
525                ),
526            })
527        } else if setup_analysis.has_custom_commands {
528            Some(SupplyChainSignal {
529                score: 15.0,
530                description: "Custom setup.py command hooks detected".to_string(),
531                evidence: vec!["setup() declares cmdclass".to_string()],
532                mitigation: Some("Review custom build/install commands before use".to_string()),
533            })
534        } else {
535            None
536        };
537
538        let risk_score = risk_calculator.calculate(
539            &vulnerabilities,
540            &malicious_patterns,
541            typosquatting_risk.is_some(),
542            supply_chain_signal,
543            50.0, // Default maintenance score
544        )?;
545
546        let risk_assessment = RiskAssessment {
547            risk_score: risk_score.clone(),
548            summary: format!(
549                "Python package '{}' has {} risk with {} advisory matches",
550                package.metadata.name.escape_default(),
551                risk_score.risk_level,
552                vulnerabilities.len()
553            ),
554            detailed_findings: vec![],
555            recommendations: vec![],
556            security_posture: crate::core::SecurityPosture {
557                vulnerability_matches_present: !vulnerabilities.is_empty(),
558                high_severity_pattern_matches_present: malicious_patterns.iter().any(|pattern| {
559                    pattern.severity >= crate::core::patterns::PatternSeverity::High
560                }),
561                supply_chain_risks: setup_analysis.code_execution_risk
562                    || setup_analysis.has_custom_commands,
563                actively_maintained: None,
564                trusted_publisher: None,
565                security_practices_score: None,
566            },
567        };
568        let vulnerability_database = self.vuln_db.metadata();
569
570        Ok(PythonAnalysisResult {
571            package,
572            risk_assessment,
573            dependency_analysis,
574            subject_vulnerabilities,
575            vulnerabilities,
576            malicious_patterns,
577            setup_analysis,
578            typosquatting_risk,
579            vulnerability_database,
580        })
581    }
582
583    fn can_analyze(&self, path: &Path) -> bool {
584        ["pyproject.toml", "setup.cfg", "setup.py"]
585            .iter()
586            .any(|file| crate::utils::input::has_regular_project_file(path, Path::new(file)))
587    }
588
589    fn name(&self) -> &str {
590        "Python Package Analyzer"
591    }
592
593    fn supported_extensions(&self) -> Vec<&str> {
594        vec![]
595    }
596}
597
598fn validate_python_metadata(metadata: &PackageMetadata) -> Result<()> {
599    for (field, value) in [
600        ("name", Some(metadata.name.as_str())),
601        ("version", Some(metadata.version.as_str())),
602        ("description", metadata.description.as_deref()),
603        ("author", metadata.author.as_deref()),
604        ("license", metadata.license.as_deref()),
605        ("homepage", metadata.homepage.as_deref()),
606        ("repository", metadata.repository.as_deref()),
607    ] {
608        if let Some(value) = value
609            && value.len() > crate::limits::MAX_METADATA_FIELD_BYTES
610        {
611            anyhow::bail!("Python package {field} exceeds the configured field limit");
612        }
613    }
614    if metadata.name.trim().is_empty() || metadata.version.trim().is_empty() {
615        anyhow::bail!("Python package name and version must not be empty");
616    }
617    if !is_python_identifier(&metadata.name) {
618        anyhow::bail!("invalid Python distribution name");
619    }
620    if metadata.keywords.len() > crate::limits::MAX_METADATA_LIST_ITEMS
621        || metadata
622            .keywords
623            .iter()
624            .any(|value| value.len() > crate::limits::MAX_METADATA_FIELD_BYTES)
625    {
626        anyhow::bail!("Python package keywords exceed the configured limits");
627    }
628    crate::utils::version_parser::parse_python_subject_version(&metadata.version)
629        .map_err(|_| anyhow::anyhow!("Python package version must be an exact PEP 440 version"))?;
630    Ok(())
631}
632
633fn optional_toml_string(
634    table: &toml::map::Map<String, toml::Value>,
635    field: &str,
636) -> Result<Option<String>> {
637    table
638        .get(field)
639        .map(|value| {
640            value
641                .as_str()
642                .map(String::from)
643                .ok_or_else(|| anyhow::anyhow!("pyproject project.{field} must be a string"))
644        })
645        .transpose()
646}
647
648fn parse_toml_string_array(value: Option<&toml::Value>, field: &str) -> Result<Vec<String>> {
649    let Some(value) = value else {
650        return Ok(vec![]);
651    };
652    let array = value
653        .as_array()
654        .ok_or_else(|| anyhow::anyhow!("{field} must be an array of strings"))?;
655    if array.len() > crate::limits::MAX_METADATA_LIST_ITEMS {
656        anyhow::bail!("{field} exceeds the configured item limit");
657    }
658    array
659        .iter()
660        .map(|value| {
661            value
662                .as_str()
663                .map(String::from)
664                .ok_or_else(|| anyhow::anyhow!("{field} entries must be strings"))
665        })
666        .collect()
667}
668
669fn parse_pyproject_authors(value: Option<&toml::Value>) -> Result<Option<String>> {
670    let Some(value) = value else {
671        return Ok(None);
672    };
673    let authors = value
674        .as_array()
675        .ok_or_else(|| anyhow::anyhow!("project.authors must be an array"))?;
676    if authors.len() > crate::limits::MAX_METADATA_LIST_ITEMS {
677        anyhow::bail!("project.authors exceeds the configured item limit");
678    }
679    let mut rendered = Vec::with_capacity(authors.len());
680    for author in authors {
681        let table = author
682            .as_table()
683            .ok_or_else(|| anyhow::anyhow!("project.authors entries must be tables"))?;
684        let name = optional_toml_string(table, "name")?;
685        let email = optional_toml_string(table, "email")?;
686        if name.is_none() && email.is_none() {
687            anyhow::bail!("project.authors entry has neither name nor email");
688        }
689        rendered.push(
690            [name, email]
691                .into_iter()
692                .flatten()
693                .collect::<Vec<_>>()
694                .join(" | "),
695        );
696    }
697    Ok((!rendered.is_empty()).then(|| rendered.join("; ")))
698}
699
700fn parse_pyproject_license(value: Option<&toml::Value>) -> Result<Option<String>> {
701    let Some(value) = value else {
702        return Ok(None);
703    };
704    if let Some(license) = value.as_str() {
705        return Ok(Some(license.to_string()));
706    }
707    let table = value
708        .as_table()
709        .ok_or_else(|| anyhow::anyhow!("project.license must be a string or table"))?;
710    if let Some(text) = table.get("text").and_then(toml::Value::as_str) {
711        return Ok(Some(text.to_string()));
712    }
713    if table.get("file").and_then(toml::Value::as_str).is_some() {
714        // Do not follow another project-controlled path merely to label a
715        // package. Preserve that the license is file-declared.
716        return Ok(Some("file-declared".to_string()));
717    }
718    anyhow::bail!("project.license table needs a string text or file field")
719}
720
721fn parse_project_url(
722    project: &toml::map::Map<String, toml::Value>,
723    label: &str,
724) -> Result<Option<String>> {
725    let Some(urls) = project.get("urls") else {
726        return Ok(None);
727    };
728    let urls = urls
729        .as_table()
730        .ok_or_else(|| anyhow::anyhow!("project.urls must be a table"))?;
731    let value = urls
732        .iter()
733        .find(|(key, _)| key.eq_ignore_ascii_case(label))
734        .map(|(_, value)| value);
735    value
736        .map(|value| {
737            value
738                .as_str()
739                .map(String::from)
740                .ok_or_else(|| anyhow::anyhow!("project.urls values must be strings"))
741        })
742        .transpose()
743}
744
745#[derive(Clone, Copy)]
746enum PythonRequirementSource {
747    Pep621,
748    RequirementsFile,
749}
750
751async fn load_python_requirements(path: &Path) -> Result<(Vec<String>, PythonRequirementSource)> {
752    if std::fs::symlink_metadata(path.join("pyproject.toml")).is_ok() {
753        let content = crate::utils::input::read_project_file(
754            path,
755            Path::new("pyproject.toml"),
756            crate::limits::MAX_PROJECT_FILE_BYTES,
757        )
758        .await?;
759        let document = parse_toml_document(&content, "pyproject.toml")?;
760        let dependencies = document
761            .get("project")
762            .and_then(|project| project.get("dependencies"));
763        if dependencies.is_some() {
764            return Ok((
765                parse_toml_string_array(dependencies, "project.dependencies")?,
766                PythonRequirementSource::Pep621,
767            ));
768        }
769    }
770
771    if std::fs::symlink_metadata(path.join("requirements.txt")).is_ok() {
772        let content = crate::utils::input::read_project_file(
773            path,
774            Path::new("requirements.txt"),
775            crate::limits::MAX_PROJECT_FILE_BYTES,
776        )
777        .await?;
778        let mut requirements = Vec::new();
779        for line in content.lines() {
780            let line = line.trim();
781            if line.is_empty() || line.starts_with('#') {
782                continue;
783            }
784            let line = strip_python_requirement_comment(line).trim();
785            if !line.is_empty() {
786                if requirements.len() == crate::limits::MAX_DIRECT_DEPENDENCIES {
787                    anyhow::bail!("dependency count exceeds the configured limit");
788                }
789                requirements.push(line.to_string());
790            }
791        }
792        return Ok((requirements, PythonRequirementSource::RequirementsFile));
793    }
794
795    Ok((vec![], PythonRequirementSource::RequirementsFile))
796}
797
798fn parse_python_requirement(requirement: &str) -> Option<(String, String, bool)> {
799    let (requirement, has_environment_marker) = requirement
800        .split_once(';')
801        .map_or((requirement, false), |(base, _)| (base, true));
802    let requirement = requirement.trim();
803    if requirement.is_empty() || requirement.starts_with('-') {
804        return None;
805    }
806
807    let name_end = requirement
808        .find(['[', '<', '>', '=', '!', '~', '@'])
809        .unwrap_or(requirement.len());
810    let name = requirement[..name_end].trim();
811    if !is_python_identifier(name) {
812        return None;
813    }
814
815    let after_name = &requirement[name_end..];
816    let after_extras = if let Some(extras) = after_name.strip_prefix('[') {
817        let closing = extras.find(']')?;
818        if !extras[..closing]
819            .split(',')
820            .map(str::trim)
821            .all(is_python_identifier)
822        {
823            return None;
824        }
825        &extras[closing + 1..]
826    } else {
827        after_name
828    };
829    let version_spec = after_extras.trim();
830    let version_spec = if version_spec.is_empty() {
831        "*"
832    } else {
833        version_spec
834    };
835    Some((
836        name.to_string(),
837        version_spec.to_string(),
838        has_environment_marker,
839    ))
840}
841
842fn strip_python_requirement_comment(line: &str) -> &str {
843    line.char_indices()
844        .find(|(index, character)| {
845            *character == '#'
846                && *index > 0
847                && line[..*index]
848                    .chars()
849                    .next_back()
850                    .is_some_and(char::is_whitespace)
851        })
852        .map_or(line, |(index, _)| &line[..index])
853}
854
855fn parse_toml_document(content: &str, label: &str) -> Result<toml::Value> {
856    toml::from_str(content).map_err(|error: toml::de::Error| {
857        error.span().map_or_else(
858            || anyhow::anyhow!("{label} contains invalid TOML"),
859            |span| {
860                anyhow::anyhow!(
861                    "{label} contains invalid TOML near byte range {}..{}",
862                    span.start,
863                    span.end
864                )
865            },
866        )
867    })
868}
869
870fn is_python_identifier(value: &str) -> bool {
871    value
872        .bytes()
873        .next()
874        .is_some_and(|byte| byte.is_ascii_alphanumeric())
875        && value
876            .bytes()
877            .last()
878            .is_some_and(|byte| byte.is_ascii_alphanumeric())
879        && value
880            .bytes()
881            .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-'))
882}
883
884fn contains_python_identifier(content: &str, identifier: &str) -> bool {
885    content.match_indices(identifier).any(|(offset, _)| {
886        let before = content[..offset].chars().next_back();
887        let after = content[offset + identifier.len()..].chars().next();
888        !before.is_some_and(|character| character.is_alphanumeric() || character == '_')
889            && !after.is_some_and(|character| character.is_alphanumeric() || character == '_')
890    })
891}
892
893fn contains_python_function_call(content: &str, function: &str) -> bool {
894    content.match_indices(function).any(|(offset, _)| {
895        let before = content[..offset].chars().next_back();
896        let after = content[offset + function.len()..]
897            .trim_start()
898            .chars()
899            .next();
900        !before.is_some_and(|character| character.is_alphanumeric() || character == '_')
901            && after == Some('(')
902    })
903}
904
905fn extract_setup_literal_fields(content: &str) -> Result<BTreeMap<String, String>> {
906    const SUPPORTED_FIELDS: [&str; 6] =
907        ["name", "version", "description", "author", "license", "url"];
908
909    let code = sanitize_python_source(content);
910    let bytes = code.as_bytes();
911    let mut candidates = Vec::new();
912    let mut search_from = 0usize;
913
914    while let Some(relative) = code[search_from..].find("setup") {
915        let start = search_from + relative;
916        let end = start + "setup".len();
917        search_from = end;
918        if start > 0 && (bytes[start - 1].is_ascii_alphanumeric() || bytes[start - 1] == b'_') {
919            continue;
920        }
921        if bytes
922            .get(end)
923            .is_some_and(|byte| byte.is_ascii_alphanumeric() || *byte == b'_')
924        {
925            continue;
926        }
927        if previous_identifier(&code[..start]).as_deref() == Some("def") {
928            continue;
929        }
930
931        let open = skip_ascii_whitespace(bytes, end);
932        if bytes.get(open) != Some(&b'(') {
933            continue;
934        }
935        let Some(close) = matching_call_parenthesis(bytes, open) else {
936            anyhow::bail!("setup.py contains an unterminated setup() call");
937        };
938        search_from = close.saturating_add(1);
939
940        let mut fields = BTreeMap::new();
941        for field in SUPPORTED_FIELDS {
942            if let Some(value) = find_setup_keyword_literal(content, &code, open, close, field)? {
943                fields.insert(field.to_string(), value);
944            }
945        }
946        if fields.contains_key("name") && fields.contains_key("version") {
947            candidates.push(fields);
948        }
949    }
950
951    match candidates.len() {
952        1 => Ok(candidates.pop().unwrap_or_default()),
953        0 => anyhow::bail!("setup.py has no unambiguous setup() call with static name and version"),
954        _ => anyhow::bail!("setup.py has multiple setup() calls with static name and version"),
955    }
956}
957
958fn previous_identifier(value: &str) -> Option<String> {
959    let value = value.trim_end();
960    let start = value
961        .rfind(|character: char| !(character.is_ascii_alphanumeric() || character == '_'))
962        .map_or(0, |index| index + 1);
963    (!value[start..].is_empty()).then(|| value[start..].to_string())
964}
965
966fn matching_call_parenthesis(bytes: &[u8], open: usize) -> Option<usize> {
967    let mut parenthesis_depth = 0usize;
968    let mut bracket_depth = 0usize;
969    let mut brace_depth = 0usize;
970    for (index, byte) in bytes.iter().copied().enumerate().skip(open) {
971        match byte {
972            b'(' => parenthesis_depth = parenthesis_depth.checked_add(1)?,
973            b')' if parenthesis_depth == 1 && bracket_depth == 0 && brace_depth == 0 => {
974                return Some(index);
975            }
976            b')' => parenthesis_depth = parenthesis_depth.checked_sub(1)?,
977            b'[' => bracket_depth = bracket_depth.checked_add(1)?,
978            b']' => bracket_depth = bracket_depth.checked_sub(1)?,
979            b'{' => brace_depth = brace_depth.checked_add(1)?,
980            b'}' => brace_depth = brace_depth.checked_sub(1)?,
981            _ => {}
982        }
983    }
984    None
985}
986
987fn find_setup_keyword_literal(
988    raw: &str,
989    code: &str,
990    open: usize,
991    close: usize,
992    field: &str,
993) -> Result<Option<String>> {
994    let bytes = code.as_bytes();
995    let mut parenthesis_depth = 1usize;
996    let mut bracket_depth = 0usize;
997    let mut brace_depth = 0usize;
998    let mut found = None;
999    let mut index = open + 1;
1000
1001    while index < close {
1002        match bytes[index] {
1003            b'(' => parenthesis_depth += 1,
1004            b')' => parenthesis_depth = parenthesis_depth.saturating_sub(1),
1005            b'[' => bracket_depth += 1,
1006            b']' => bracket_depth = bracket_depth.saturating_sub(1),
1007            b'{' => brace_depth += 1,
1008            b'}' => brace_depth = brace_depth.saturating_sub(1),
1009            _ => {}
1010        }
1011
1012        if parenthesis_depth == 1
1013            && bracket_depth == 0
1014            && brace_depth == 0
1015            && bytes[index..].starts_with(field.as_bytes())
1016            && (index == open + 1
1017                || !(bytes[index - 1].is_ascii_alphanumeric() || bytes[index - 1] == b'_'))
1018            && bytes
1019                .get(index + field.len())
1020                .is_none_or(|byte| !byte.is_ascii_alphanumeric() && *byte != b'_')
1021        {
1022            let equals = skip_ascii_whitespace(bytes, index + field.len());
1023            if bytes.get(equals) == Some(&b'=') && bytes.get(equals + 1) != Some(&b'=') {
1024                let value_start = skip_ascii_whitespace(raw.as_bytes(), equals + 1);
1025                let Some((value, literal_end)) = parse_static_python_string(raw, value_start)
1026                else {
1027                    anyhow::bail!("setup() {field} must be a simple static string literal");
1028                };
1029                let value_end = skip_ascii_whitespace(bytes, literal_end);
1030                if value_end > close
1031                    || !only_python_spacing_or_comments(&raw[literal_end..value_end])
1032                    || !matches!(bytes.get(value_end), Some(b',') | Some(b')'))
1033                {
1034                    anyhow::bail!("setup() {field} must be exactly one static string literal");
1035                }
1036                if found.replace(value).is_some() {
1037                    anyhow::bail!("setup() contains duplicate {field} fields");
1038                }
1039            }
1040        }
1041        index += 1;
1042    }
1043
1044    Ok(found)
1045}
1046
1047fn skip_ascii_whitespace(bytes: &[u8], mut index: usize) -> usize {
1048    while bytes.get(index).is_some_and(u8::is_ascii_whitespace) {
1049        index += 1;
1050    }
1051    index
1052}
1053
1054fn only_python_spacing_or_comments(value: &str) -> bool {
1055    let bytes = value.as_bytes();
1056    let mut in_comment = false;
1057    for byte in bytes {
1058        if in_comment {
1059            if matches!(byte, b'\n' | b'\r') {
1060                in_comment = false;
1061            }
1062        } else if *byte == b'#' {
1063            in_comment = true;
1064        } else if !byte.is_ascii_whitespace() {
1065            return false;
1066        }
1067    }
1068    true
1069}
1070
1071fn parse_static_python_string(content: &str, start: usize) -> Option<(String, usize)> {
1072    let bytes = content.as_bytes();
1073    let mut quote_index = start;
1074    while bytes
1075        .get(quote_index)
1076        .is_some_and(|byte| byte.is_ascii_alphabetic())
1077    {
1078        quote_index += 1;
1079    }
1080    let prefix = content.get(start..quote_index)?.to_ascii_lowercase();
1081    if !matches!(prefix.as_str(), "" | "r" | "u" | "ru" | "ur") {
1082        return None;
1083    }
1084    let quote = *bytes.get(quote_index)?;
1085    if !matches!(quote, b'\'' | b'"')
1086        || bytes.get(quote_index + 1) == Some(&quote)
1087        || bytes.get(quote_index + 2) == Some(&quote)
1088    {
1089        return None;
1090    }
1091    let mut index = quote_index + 1;
1092    while let Some(byte) = bytes.get(index).copied() {
1093        if byte == quote {
1094            return Some((content[quote_index + 1..index].to_string(), index + 1));
1095        }
1096        // Reject escapes instead of guessing at Python literal semantics.
1097        if byte == b'\\' || matches!(byte, b'\n' | b'\r') {
1098            return None;
1099        }
1100        index += 1;
1101    }
1102    None
1103}
1104
1105#[derive(Clone, Copy)]
1106enum PythonLexState {
1107    Code,
1108    Comment,
1109    String {
1110        quote: u8,
1111        triple: bool,
1112        formatted: bool,
1113    },
1114    FormattedExpression {
1115        brace_depth: usize,
1116    },
1117}
1118
1119/// Remove comments and string contents while preserving executable tokens and
1120/// line boundaries. This is a lexical filter, not Python execution or an AST.
1121fn sanitize_python_source(content: &str) -> String {
1122    let bytes = content.as_bytes();
1123    let mut output = Vec::with_capacity(bytes.len());
1124    let mut states = vec![PythonLexState::Code];
1125    let mut index = 0usize;
1126
1127    while index < bytes.len() {
1128        match *states.last().unwrap_or(&PythonLexState::Code) {
1129            PythonLexState::Code => match bytes[index] {
1130                b'#' => {
1131                    output.push(b' ');
1132                    index += 1;
1133                    states.push(PythonLexState::Comment);
1134                }
1135                quote @ (b'\'' | b'"') => {
1136                    let triple = bytes.get(index + 1) == Some(&quote)
1137                        && bytes.get(index + 2) == Some(&quote);
1138                    let width = if triple { 3 } else { 1 };
1139                    output.extend(std::iter::repeat_n(b' ', width));
1140                    let formatted = has_formatted_string_prefix(bytes, index);
1141                    index += width;
1142                    states.push(PythonLexState::String {
1143                        quote,
1144                        triple,
1145                        formatted,
1146                    });
1147                }
1148                byte => {
1149                    output.push(byte);
1150                    index += 1;
1151                }
1152            },
1153            PythonLexState::Comment => {
1154                let byte = bytes[index];
1155                if matches!(byte, b'\n' | b'\r') {
1156                    output.push(byte);
1157                    states.pop();
1158                } else {
1159                    output.push(b' ');
1160                }
1161                index += 1;
1162            }
1163            PythonLexState::String {
1164                quote,
1165                triple,
1166                formatted,
1167            } => {
1168                if bytes[index] == b'\\' {
1169                    output.push(b' ');
1170                    index += 1;
1171                    if let Some(byte) = bytes.get(index).copied() {
1172                        output.push(if matches!(byte, b'\n' | b'\r') {
1173                            byte
1174                        } else {
1175                            b' '
1176                        });
1177                        index += 1;
1178                    }
1179                } else if triple
1180                    && bytes.get(index) == Some(&quote)
1181                    && bytes.get(index + 1) == Some(&quote)
1182                    && bytes.get(index + 2) == Some(&quote)
1183                {
1184                    output.extend_from_slice(b"   ");
1185                    index += 3;
1186                    states.pop();
1187                } else if !triple && bytes[index] == quote {
1188                    output.push(b' ');
1189                    index += 1;
1190                    states.pop();
1191                } else if formatted && bytes[index] == b'{' && bytes.get(index + 1) != Some(&b'{') {
1192                    output.push(b' ');
1193                    index += 1;
1194                    states.push(PythonLexState::FormattedExpression { brace_depth: 1 });
1195                } else if formatted
1196                    && matches!(bytes[index], b'{' | b'}')
1197                    && bytes.get(index + 1) == Some(&bytes[index])
1198                {
1199                    output.extend_from_slice(b"  ");
1200                    index += 2;
1201                } else {
1202                    let byte = bytes[index];
1203                    output.push(if matches!(byte, b'\n' | b'\r') {
1204                        byte
1205                    } else {
1206                        b' '
1207                    });
1208                    index += 1;
1209                }
1210            }
1211            PythonLexState::FormattedExpression { brace_depth } => match bytes[index] {
1212                b'#' => {
1213                    output.push(b' ');
1214                    index += 1;
1215                    states.push(PythonLexState::Comment);
1216                }
1217                quote @ (b'\'' | b'"') => {
1218                    let triple = bytes.get(index + 1) == Some(&quote)
1219                        && bytes.get(index + 2) == Some(&quote);
1220                    let width = if triple { 3 } else { 1 };
1221                    output.extend(std::iter::repeat_n(b' ', width));
1222                    let formatted = has_formatted_string_prefix(bytes, index);
1223                    index += width;
1224                    states.push(PythonLexState::String {
1225                        quote,
1226                        triple,
1227                        formatted,
1228                    });
1229                }
1230                b'{' => {
1231                    output.push(b'{');
1232                    index += 1;
1233                    if let Some(PythonLexState::FormattedExpression { brace_depth }) =
1234                        states.last_mut()
1235                    {
1236                        *brace_depth = brace_depth.saturating_add(1);
1237                    }
1238                }
1239                b'}' if brace_depth == 1 => {
1240                    output.push(b' ');
1241                    index += 1;
1242                    states.pop();
1243                }
1244                b'}' => {
1245                    output.push(b'}');
1246                    index += 1;
1247                    if let Some(PythonLexState::FormattedExpression { brace_depth }) =
1248                        states.last_mut()
1249                    {
1250                        *brace_depth = brace_depth.saturating_sub(1);
1251                    }
1252                }
1253                byte => {
1254                    output.push(byte);
1255                    index += 1;
1256                }
1257            },
1258        }
1259    }
1260
1261    String::from_utf8(output).unwrap_or_default()
1262}
1263
1264fn has_formatted_string_prefix(bytes: &[u8], quote_index: usize) -> bool {
1265    let mut start = quote_index;
1266    while start > 0 && bytes[start - 1].is_ascii_alphabetic() {
1267        start -= 1;
1268    }
1269    if start > 0 && (bytes[start - 1].is_ascii_alphanumeric() || bytes[start - 1] == b'_') {
1270        return false;
1271    }
1272    matches!(
1273        bytes[start..quote_index]
1274            .iter()
1275            .map(u8::to_ascii_lowercase)
1276            .collect::<Vec<_>>()
1277            .as_slice(),
1278        b"f" | b"fr" | b"rf"
1279    )
1280}