1use std::collections::{BTreeMap, BTreeSet};
7use std::path::Path;
8use std::sync::LazyLock;
9
10use serde::{Deserialize, Serialize};
11use sqlparser::ast::{
12 AlterTableOperation, ColumnDef, ColumnOption, Expr, FromTable, IndexColumn, ObjectName, Query, SetExpr, Statement, TableConstraint, TableFactor, TableObject, TableWithJoins
13};
14use sqlparser::dialect::{GenericDialect, MySqlDialect};
15use sqlparser::parser::Parser;
16use thiserror::Error;
17use tree_sitter::Node as SyntaxNode;
18
19use crate::SourceLanguage;
20use crate::markers::MarkerSet;
21
22pub const MAX_DATA_INPUT_BYTES: usize = 1_048_576;
24pub const MAX_DATA_SOURCE_LINES: u32 = 100_000;
26pub const MAX_DATA_ITEMS: usize = 4_096;
28pub const MAX_DATA_DEPTH: usize = 32;
30
31const MAX_IDENTIFIER_CHARS: usize = 512;
32const MAX_LITERAL_SQL_BYTES: usize = 65_536;
33
34#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
36#[serde(rename_all = "snake_case")]
37pub enum DataArtifactKind {
38 SqlMigration,
40 DeclarativeSqlSchema,
42 Prisma,
44 Alembic,
46 SqlAlchemy,
48 Diesel,
50 SqlQueryFile,
52 SqlxConfiguration,
54 LiteralQuerySource,
56}
57
58#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
60#[serde(rename_all = "snake_case")]
61pub enum DataFramework {
62 Sqlx,
64 MysqlAsync,
66 PyMysql,
68 Psycopg,
70 SqlAlchemy,
72}
73
74#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
76#[serde(rename_all = "snake_case")]
77pub enum DataArtifactReferenceKind {
78 QueryFile,
80 MigrationDirectory,
82 SqlxDefaultMigrationDirectory,
84}
85
86#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
88#[serde(rename_all = "snake_case")]
89pub enum DataAccessRole {
90 Reader,
92 Writer,
94 ModelBinding,
96 Declaration,
98}
99
100#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
102#[serde(rename_all = "snake_case")]
103pub enum DataOperation {
104 Select,
106 Insert,
108 Update,
110 Delete,
112}
113
114#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
116#[serde(rename_all = "snake_case")]
117pub enum DataWarning {
118 DynamicQuery,
120 UnsupportedConstruct,
122 UnresolvedReference,
124 LimitExceeded,
126 SqlParseRecovery,
128}
129
130#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
132#[serde(transparent)]
133pub struct DataEvidenceLine(u32);
134
135impl DataEvidenceLine {
136 pub fn new(line: u32) -> Result<Self, DataExtractionError> {
143 if line == 0 || line > MAX_DATA_SOURCE_LINES {
144 return Err(DataExtractionError::InvalidEvidenceLine { line });
145 }
146 Ok(Self(line))
147 }
148
149 #[must_use]
151 pub const fn get(self) -> u32 {
152 self.0
153 }
154}
155
156#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
158pub struct DatabaseColumn {
159 pub name: String,
161 pub data_type: Option<String>,
163 pub nullable: Option<bool>,
165 pub primary_key: bool,
167 pub unique: bool,
169 pub default_present: bool,
171 pub evidence: DataEvidenceLine,
173}
174
175#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
177pub struct DatabaseIndex {
178 pub name: Option<String>,
180 pub columns: Vec<String>,
182 pub unique: bool,
184 pub evidence: DataEvidenceLine,
186}
187
188#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
190pub struct DatabaseForeignKey {
191 pub name: Option<String>,
193 pub columns: Vec<String>,
195 pub referenced_table: String,
197 pub referenced_columns: Vec<String>,
199 pub evidence: DataEvidenceLine,
201}
202
203#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
205pub struct DatabaseTable {
206 pub database: Option<String>,
208 pub schema: Option<String>,
210 pub name: String,
212 pub columns: Vec<DatabaseColumn>,
214 pub indexes: Vec<DatabaseIndex>,
216 pub foreign_keys: Vec<DatabaseForeignKey>,
218 pub evidence: DataEvidenceLine,
220}
221
222#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
224pub struct MigrationMetadata {
225 pub revision: Option<String>,
227 pub down_revision: Option<String>,
229 pub order_hint: Option<u64>,
231 pub reversible: bool,
233 pub evidence: DataEvidenceLine,
235}
236
237#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
239pub struct DataAccessObservation {
240 pub role: DataAccessRole,
242 pub table: String,
244 pub model: Option<String>,
246 pub owner: Option<String>,
248 pub operation: Option<DataOperation>,
250 pub evidence: DataEvidenceLine,
252}
253
254#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
256pub struct DataArtifactReference {
257 pub framework: DataFramework,
259 pub kind: DataArtifactReferenceKind,
261 pub path: String,
263 pub owner: Option<String>,
265 pub evidence: DataEvidenceLine,
267}
268
269#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
271pub struct DataDocument {
272 pub source_path: String,
274 pub artifact_kind: DataArtifactKind,
276 pub database_name: Option<String>,
278 pub schema_name: Option<String>,
280 pub tables: Vec<DatabaseTable>,
282 pub migration: Option<MigrationMetadata>,
284 pub accesses: Vec<DataAccessObservation>,
286 pub frameworks: Vec<DataFramework>,
288 pub references: Vec<DataArtifactReference>,
290 pub owners: Vec<String>,
292 pub warnings: Vec<DataWarning>,
294 pub incomplete: bool,
296}
297
298#[derive(Debug, Error, PartialEq, Eq)]
300pub enum DataExtractionError {
301 #[error("database artifact input exceeds the byte limit ({actual} > {maximum})")]
303 InputTooLarge {
304 actual: usize,
306 maximum: usize,
308 },
309 #[error("database artifact input exceeds the line limit ({actual} > {maximum})")]
311 TooManyLines {
312 actual: u32,
314 maximum: u32,
316 },
317 #[error("unsupported database artifact filename or extension")]
319 UnsupportedArtifact,
320 #[error("SQL artifact could not be parsed")]
322 SqlParseFailed,
323 #[error("database artifact exceeds the structured item limit")]
325 ItemLimitExceeded,
326 #[error("evidence line {line} is outside the supported range")]
328 InvalidEvidenceLine {
329 line: u32,
331 },
332}
333
334pub fn extract_data_artifact(
347 source_path: &str,
348 input: &str,
349) -> Result<DataDocument, DataExtractionError> {
350 let kind = artifact_kind(source_path, input).ok_or(DataExtractionError::UnsupportedArtifact)?;
351 if validate_input(input).is_err() {
352 let mut document = empty_document(source_path, kind);
353 mark_incomplete(&mut document, DataWarning::LimitExceeded);
354 return Ok(document);
355 }
356 let mut document = match kind {
357 DataArtifactKind::SqlMigration
358 | DataArtifactKind::DeclarativeSqlSchema
359 | DataArtifactKind::SqlQueryFile => parse_sql_artifact(source_path, input, kind),
360 DataArtifactKind::SqlxConfiguration => parse_sqlx_configuration(source_path, input),
361 DataArtifactKind::Prisma => parse_prisma(source_path, input),
362 DataArtifactKind::Alembic => parse_alembic(source_path, input),
363 DataArtifactKind::SqlAlchemy => parse_sqlalchemy(source_path, input),
364 DataArtifactKind::Diesel => parse_diesel(source_path, input),
365 DataArtifactKind::LiteralQuerySource => {
366 return Err(DataExtractionError::UnsupportedArtifact);
367 }
368 };
369 finish_document(&mut document);
370 Ok(document)
371}
372
373#[must_use]
379pub fn parse_literal_sql_source(
380 language: SourceLanguage,
381 source_path: &str,
382 input: &str,
383) -> DataDocument {
384 parse_literal_sql_source_at_root(
385 language,
386 source_path,
387 &inferred_crate_root(source_path),
388 input,
389 )
390}
391
392#[must_use]
397pub fn parse_literal_sql_source_at_root(
398 language: SourceLanguage,
399 source_path: &str,
400 crate_root: &str,
401 input: &str,
402) -> DataDocument {
403 parse_literal_sql_source_with_prefilter(language, source_path, crate_root, input, true)
404}
405
406fn parse_literal_sql_source_with_prefilter(
407 language: SourceLanguage,
408 source_path: &str,
409 crate_root: &str,
410 input: &str,
411 prefilter: bool,
412) -> DataDocument {
413 let mut document = empty_document(source_path, DataArtifactKind::LiteralQuerySource);
414 if validate_input(input).is_err() {
415 mark_incomplete(&mut document, DataWarning::LimitExceeded);
416 return document;
417 }
418 if prefilter && !DATA_SOURCE_MARKERS.any_in(input) {
419 finish_document(&mut document);
420 return document;
421 }
422
423 if language == SourceLanguage::Rust {
424 if !prefilter || input.contains("sqlx") || input.contains("mysql_async") {
425 extract_rust_database_source(input, crate_root, &mut document);
426 }
427 } else if language == SourceLanguage::Python {
428 if input.contains("pymysql") {
429 document.frameworks.push(DataFramework::PyMysql);
430 }
431 if input.contains("psycopg") {
432 document.frameworks.push(DataFramework::Psycopg);
433 }
434 if input.contains("sqlalchemy") || input.contains(".query(") {
435 extract_sqlalchemy_accesses(input, &mut document);
436 }
437 }
438 let literals = quoted_literals(input, language);
439 if literals.len() >= MAX_DATA_ITEMS {
440 mark_incomplete(&mut document, DataWarning::LimitExceeded);
441 }
442 for literal in literals {
443 if literal.value.len() > MAX_LITERAL_SQL_BYTES || !has_query_context(input, &literal) {
444 continue;
445 }
446 let query = if language == SourceLanguage::Python
447 && (literal.interpolated || python_format_call_after(input, &literal))
448 {
449 mark_incomplete(&mut document, DataWarning::DynamicQuery);
450 sanitize_python_f_string(&literal.value)
451 } else if literal.dynamic {
452 mark_incomplete(&mut document, DataWarning::DynamicQuery);
453 None
454 } else {
455 Some(literal.value.clone())
456 };
457 let Some(query) = query else {
458 continue;
459 };
460 let first_keyword = first_sql_keyword(&query);
461 if !matches!(
462 first_keyword.as_deref(),
463 Some("SELECT" | "WITH" | "INSERT" | "UPDATE" | "DELETE")
464 ) {
465 continue;
466 }
467 let Some(statements) = parse_sql_with_supported_dialects(&query) else {
468 mark_incomplete(&mut document, DataWarning::SqlParseRecovery);
469 continue;
470 };
471 let owner = owner_at_line(language, input, literal.line);
472 if let Some(owner) = owner.as_ref() {
473 document.owners.push(owner.clone());
474 }
475 for statement in &statements {
476 append_statement_accesses(
477 statement,
478 evidence(literal.line),
479 owner.as_deref(),
480 &mut document,
481 0,
482 );
483 }
484 document
485 .accesses
486 .retain(|access| !access.table.contains("__csg_dynamic_value__"));
487 }
488
489 if (language != SourceLanguage::Rust || document.frameworks.is_empty())
490 && source_has_dynamic_query(input, language)
491 {
492 mark_incomplete(&mut document, DataWarning::DynamicQuery);
493 }
494 finish_document(&mut document);
495 document
496}
497
498fn validate_input(input: &str) -> Result<(), DataExtractionError> {
499 if input.len() > MAX_DATA_INPUT_BYTES {
500 return Err(DataExtractionError::InputTooLarge {
501 actual: input.len(),
502 maximum: MAX_DATA_INPUT_BYTES,
503 });
504 }
505 let lines = input.lines().count().max(1);
506 let lines = u32::try_from(lines).unwrap_or(u32::MAX);
507 if lines > MAX_DATA_SOURCE_LINES {
508 return Err(DataExtractionError::TooManyLines {
509 actual: lines,
510 maximum: MAX_DATA_SOURCE_LINES,
511 });
512 }
513 Ok(())
514}
515
516fn artifact_kind(source_path: &str, input: &str) -> Option<DataArtifactKind> {
517 let path = Path::new(source_path);
518 let filename = path.file_name()?.to_str()?.to_ascii_lowercase();
519 let extension = path.extension()?.to_str()?.to_ascii_lowercase();
520 let normalized = source_path.replace('\\', "/").to_ascii_lowercase();
521 match (filename.as_str(), extension.as_str()) {
522 ("sqlx.toml", "toml") => Some(DataArtifactKind::SqlxConfiguration),
523 ("schema.sql" | "structure.sql" | "init.sql", "sql") => {
524 Some(DataArtifactKind::DeclarativeSqlSchema)
525 }
526 (_, "sql") if sql_migration_path(&normalized, &filename) => {
527 Some(DataArtifactKind::SqlMigration)
528 }
529 (_, "sql") if sql_contains_only_queries(input) => Some(DataArtifactKind::SqlQueryFile),
530 (_, "sql") => Some(DataArtifactKind::DeclarativeSqlSchema),
531 (_, "prisma") => Some(DataArtifactKind::Prisma),
532 ("schema.rs", "rs") => Some(DataArtifactKind::Diesel),
533 (_, "py")
534 if normalized.contains("/alembic/")
535 || normalized.contains("/versions/")
536 || normalized.contains("/migrations/") =>
537 {
538 Some(DataArtifactKind::Alembic)
539 }
540 ("model.py" | "models.py" | "entities.py", "py") => Some(DataArtifactKind::SqlAlchemy),
541 _ => None,
542 }
543}
544
545fn empty_document(source_path: &str, artifact_kind: DataArtifactKind) -> DataDocument {
546 DataDocument {
547 source_path: source_path.to_owned(),
548 artifact_kind,
549 database_name: None,
550 schema_name: None,
551 tables: Vec::new(),
552 migration: None,
553 accesses: Vec::new(),
554 frameworks: Vec::new(),
555 references: Vec::new(),
556 owners: Vec::new(),
557 warnings: Vec::new(),
558 incomplete: false,
559 }
560}
561
562fn parse_sql_artifact(source_path: &str, input: &str, kind: DataArtifactKind) -> DataDocument {
563 let mut document = empty_document(source_path, kind);
564 if let Some(statements) = parse_sql_with_supported_dialects(input) {
565 let starts = sql_statement_lines(input);
566 for (index, statement) in statements.iter().enumerate() {
567 append_sql_artifact_statement(
568 statement,
569 starts.get(index).copied().unwrap_or(1),
570 kind,
571 &mut document,
572 );
573 }
574 } else {
575 mark_incomplete(&mut document, DataWarning::SqlParseRecovery);
576 for chunk in sql_statement_chunks(input) {
577 if !sql_chunk_is_relevant(chunk.text, kind) {
578 continue;
579 }
580 let Some(statements) = parse_sql_with_supported_dialects(chunk.text) else {
581 continue;
582 };
583 for statement in &statements {
584 append_sql_artifact_statement(statement, chunk.line, kind, &mut document);
585 }
586 }
587 }
588
589 if kind == DataArtifactKind::SqlMigration {
590 document.migration = Some(sql_migration_metadata(source_path, input));
591 }
592 document
593}
594
595fn parse_sql_with_supported_dialects(input: &str) -> Option<Vec<Statement>> {
596 Parser::parse_sql(&GenericDialect {}, input)
597 .or_else(|_| Parser::parse_sql(&MySqlDialect {}, input))
598 .ok()
599}
600
601fn append_sql_artifact_statement(
602 statement: &Statement,
603 line: u32,
604 kind: DataArtifactKind,
605 document: &mut DataDocument,
606) {
607 if kind == DataArtifactKind::SqlQueryFile {
608 append_statement_accesses(statement, evidence(line), None, document, 0);
609 } else if !append_schema_statement(statement, evidence(line), document) {
610 mark_incomplete(document, DataWarning::UnsupportedConstruct);
611 }
612}
613
614fn sql_chunk_is_relevant(input: &str, kind: DataArtifactKind) -> bool {
615 let Some(keyword) = first_sql_keyword(input) else {
616 return false;
617 };
618 if kind == DataArtifactKind::SqlQueryFile {
619 matches!(
620 keyword.as_str(),
621 "SELECT" | "WITH" | "INSERT" | "UPDATE" | "DELETE"
622 )
623 } else {
624 matches!(keyword.as_str(), "CREATE" | "ALTER")
625 }
626}
627
628fn parse_sqlx_configuration(source_path: &str, input: &str) -> DataDocument {
629 let mut document = empty_document(source_path, DataArtifactKind::SqlxConfiguration);
630 document.frameworks.push(DataFramework::Sqlx);
631 let crate_root = Path::new(source_path)
632 .parent()
633 .and_then(Path::to_str)
634 .unwrap_or_default()
635 .replace('\\', "/");
636 let mut section = "";
637
638 for (index, source_line) in input.lines().enumerate() {
639 let line = strip_toml_comment(source_line).trim();
640 if line.starts_with('[') && line.ends_with(']') {
641 section = line[1..line.len() - 1].trim();
642 continue;
643 }
644 if section != "migrate" {
645 continue;
646 }
647 let Some((key, value)) = line.split_once('=') else {
648 continue;
649 };
650 if key.trim() != "migrations-dir" {
651 continue;
652 }
653 let Some(value) = toml_string(value.trim()) else {
654 mark_incomplete(&mut document, DataWarning::UnresolvedReference);
655 continue;
656 };
657 push_sqlx_reference(
658 DataArtifactReferenceKind::MigrationDirectory,
659 value,
660 evidence(u32::try_from(index + 1).unwrap_or(u32::MAX)),
661 &crate_root,
662 None,
663 &mut document,
664 );
665 }
666 document
667}
668
669fn strip_toml_comment(line: &str) -> &str {
670 let mut quote = None;
671 let mut escaped = false;
672 for (index, character) in line.char_indices() {
673 if escaped {
674 escaped = false;
675 continue;
676 }
677 match character {
678 '\\' if quote == Some('"') => escaped = true,
679 '\'' | '"' if quote == Some(character) => quote = None,
680 '\'' | '"' if quote.is_none() => quote = Some(character),
681 '#' if quote.is_none() => return &line[..index],
682 _ => {}
683 }
684 }
685 line
686}
687
688fn toml_string(value: &str) -> Option<&str> {
689 let quote = value.chars().next()?;
690 if !matches!(quote, '\'' | '"') || !value.ends_with(quote) || value.len() < 2 {
691 return None;
692 }
693 Some(&value[1..value.len() - 1])
694}
695
696fn append_schema_statement(
697 statement: &Statement,
698 line: DataEvidenceLine,
699 document: &mut DataDocument,
700) -> bool {
701 match statement {
702 Statement::CreateTable(create) => {
703 let mut table = table_from_name(&create.name, line);
704 table.columns = create
705 .columns
706 .iter()
707 .map(|column| sql_column(column, line))
708 .collect();
709 apply_table_constraints(&mut table, &create.constraints, line);
710 push_declaration_access(document, &table, None);
711 document.tables.push(table);
712 }
713 Statement::CreateIndex(index) => {
714 let table = ensure_table(document, &index.table_name, line);
715 table.indexes.push(DatabaseIndex {
716 name: index.name.as_ref().map(object_name),
717 columns: index.columns.iter().filter_map(index_column_name).collect(),
718 unique: index.unique,
719 evidence: line,
720 });
721 }
722 Statement::AlterTable(alter) => {
723 let table = ensure_table(document, &alter.name, line);
724 for operation in &alter.operations {
725 apply_alter_operation(table, operation, line);
726 }
727 }
728 _ => return false,
729 }
730 true
731}
732
733fn sql_column(column: &ColumnDef, line: DataEvidenceLine) -> DatabaseColumn {
734 let mut nullable = None;
735 let mut primary_key = false;
736 let mut unique = false;
737 let mut default_present = false;
738 for option in &column.options {
739 match &option.option {
740 ColumnOption::Null => nullable = Some(true),
741 ColumnOption::NotNull => nullable = Some(false),
742 ColumnOption::PrimaryKey(_) => primary_key = true,
743 ColumnOption::Unique(_) => unique = true,
744 ColumnOption::Default(_)
745 | ColumnOption::Materialized(_)
746 | ColumnOption::Generated { .. }
747 | ColumnOption::Identity(_) => default_present = true,
748 _ => {}
749 }
750 }
751 DatabaseColumn {
752 name: bounded_identifier(&column.name.value),
753 data_type: Some(safe_data_type(&column.data_type.to_string())),
754 nullable,
755 primary_key,
756 unique,
757 default_present,
758 evidence: line,
759 }
760}
761
762fn apply_table_constraints(
763 table: &mut DatabaseTable,
764 constraints: &[TableConstraint],
765 line: DataEvidenceLine,
766) {
767 for constraint in constraints {
768 match constraint {
769 TableConstraint::PrimaryKey(primary) => {
770 let columns = primary
771 .columns
772 .iter()
773 .filter_map(index_column_name)
774 .collect::<Vec<_>>();
775 mark_columns(table, &columns, true, false);
776 }
777 TableConstraint::Unique(unique) => {
778 let columns = unique
779 .columns
780 .iter()
781 .filter_map(index_column_name)
782 .collect::<Vec<_>>();
783 mark_columns(table, &columns, false, true);
784 table.indexes.push(DatabaseIndex {
785 name: unique
786 .name
787 .as_ref()
788 .or(unique.index_name.as_ref())
789 .map(|name| bounded_identifier(&name.value)),
790 columns,
791 unique: true,
792 evidence: line,
793 });
794 }
795 TableConstraint::ForeignKey(foreign) => {
796 table.foreign_keys.push(DatabaseForeignKey {
797 name: foreign
798 .name
799 .as_ref()
800 .map(|name| bounded_identifier(&name.value)),
801 columns: foreign
802 .columns
803 .iter()
804 .map(|name| bounded_identifier(&name.value))
805 .collect(),
806 referenced_table: object_name(&foreign.foreign_table),
807 referenced_columns: foreign
808 .referred_columns
809 .iter()
810 .map(|name| bounded_identifier(&name.value))
811 .collect(),
812 evidence: line,
813 });
814 }
815 TableConstraint::Index(index) => {
816 table.indexes.push(DatabaseIndex {
817 name: index
818 .name
819 .as_ref()
820 .map(|name| bounded_identifier(&name.value)),
821 columns: index.columns.iter().filter_map(index_column_name).collect(),
822 unique: false,
823 evidence: line,
824 });
825 }
826 _ => {}
827 }
828 }
829}
830
831fn apply_alter_operation(
832 table: &mut DatabaseTable,
833 operation: &AlterTableOperation,
834 line: DataEvidenceLine,
835) {
836 match operation {
837 AlterTableOperation::AddColumn { column_def, .. } => {
838 table.columns.push(sql_column(column_def, line));
839 }
840 AlterTableOperation::AddConstraint { constraint, .. } => {
841 apply_table_constraints(table, std::slice::from_ref(constraint), line);
842 }
843 _ => {}
844 }
845}
846
847fn parse_prisma(source_path: &str, input: &str) -> DataDocument {
848 let mut document = empty_document(source_path, DataArtifactKind::Prisma);
849 for block in named_blocks(input, "model") {
850 let model = bounded_identifier(block.name);
851 let mapped = find_call_literal(block.body, "@@map").unwrap_or_else(|| model.clone());
852 let schema = find_call_literal(block.body, "@@schema");
853 let mut table = table_from_qualified_text(&mapped, evidence(block.line));
854 table.schema = schema.map(|value| bounded_identifier(&value));
855
856 for (offset, raw_line) in block.body.lines().enumerate() {
857 let line = add_line(block.line, offset);
858 let trimmed = raw_line.trim();
859 if trimmed.is_empty()
860 || trimmed.starts_with("//")
861 || trimmed.starts_with("@@")
862 || trimmed.starts_with('}')
863 {
864 continue;
865 }
866 let fields = trimmed.split_whitespace().collect::<Vec<_>>();
867 if fields.len() < 2 {
868 continue;
869 }
870 let field_name =
871 find_call_literal(trimmed, "@map").unwrap_or_else(|| bounded_identifier(fields[0]));
872 let field_type = fields[1];
873 if field_type.starts_with(char::is_uppercase) && trimmed.contains("@relation") {
874 if let Some(foreign_key) = prisma_foreign_key(trimmed, field_type, evidence(line)) {
875 table.foreign_keys.push(foreign_key);
876 }
877 continue;
878 }
879 table.columns.push(DatabaseColumn {
880 name: bounded_identifier(&field_name),
881 data_type: Some(bounded_identifier(
882 field_type.trim_end_matches('?').trim_end_matches("[]"),
883 )),
884 nullable: Some(field_type.ends_with('?')),
885 primary_key: trimmed.contains("@id"),
886 unique: trimmed.contains("@unique"),
887 default_present: trimmed.contains("@default("),
888 evidence: evidence(line),
889 });
890 }
891 document.owners.push(model.clone());
892 document.accesses.push(DataAccessObservation {
893 role: DataAccessRole::ModelBinding,
894 table: qualified_table_name(&table),
895 model: None,
896 owner: Some(model),
897 operation: None,
898 evidence: table.evidence,
899 });
900 document.tables.push(table);
901 }
902 document
903}
904
905fn prisma_foreign_key(
906 line: &str,
907 referenced_model: &str,
908 evidence: DataEvidenceLine,
909) -> Option<DatabaseForeignKey> {
910 let fields = bracket_values_after(line, "fields:")?;
911 let references = bracket_values_after(line, "references:").unwrap_or_default();
912 Some(DatabaseForeignKey {
913 name: None,
914 columns: fields,
915 referenced_table: bounded_identifier(referenced_model.trim_end_matches('?')),
916 referenced_columns: references,
917 evidence,
918 })
919}
920
921fn parse_alembic(source_path: &str, input: &str) -> DataDocument {
922 let mut document = empty_document(source_path, DataArtifactKind::Alembic);
923 document.migration = Some(MigrationMetadata {
924 revision: assignment_literal(input, "revision"),
925 down_revision: assignment_literal(input, "down_revision"),
926 order_hint: filename_order_hint(source_path),
927 reversible: input
928 .lines()
929 .any(|line| line.trim_start().starts_with("def downgrade(")),
930 evidence: evidence(assignment_line(input, "revision").unwrap_or(1)),
931 });
932
933 for call in collect_calls(input, "op.create_table(") {
934 let Some(name) = first_quoted(&call.body) else {
935 mark_incomplete(&mut document, DataWarning::DynamicQuery);
936 continue;
937 };
938 let mut table = table_from_qualified_text(&name, evidence(call.line));
939 for column_call in collect_calls(&call.body, "sa.Column(") {
940 if let Some(column) =
941 python_column(&column_call.body, add_evidence(call.line, column_call.line))
942 {
943 table.columns.push(column);
944 }
945 }
946 for foreign_call in collect_calls(&call.body, "sa.ForeignKeyConstraint(") {
947 if let Some(foreign) = python_foreign_key(
948 &foreign_call.body,
949 add_evidence(call.line, foreign_call.line),
950 ) {
951 table.foreign_keys.push(foreign);
952 }
953 }
954 push_declaration_access(&mut document, &table, Some("upgrade"));
955 document.tables.push(table);
956 }
957
958 for call in collect_calls(input, "op.add_column(") {
959 let quoted = quoted_values(&call.body);
960 let Some(table_name) = quoted.first() else {
961 mark_incomplete(&mut document, DataWarning::DynamicQuery);
962 continue;
963 };
964 let table = ensure_text_table(&mut document, table_name, evidence(call.line));
965 if let Some(column_call) = collect_calls(&call.body, "sa.Column(").first()
966 && let Some(column) = python_column(&column_call.body, evidence(call.line))
967 {
968 table.columns.push(column);
969 }
970 }
971
972 for call in collect_calls(input, "op.create_index(") {
973 let quoted = quoted_values(&call.body);
974 if quoted.len() < 2 {
975 mark_incomplete(&mut document, DataWarning::DynamicQuery);
976 continue;
977 }
978 let columns = bracket_values(&call.body).unwrap_or_default();
979 let unique = call.body.split_whitespace().any(|part| {
980 part.trim_matches(|character: char| character == ',' || character == ')')
981 == "unique=True"
982 });
983 let table = ensure_text_table(&mut document, "ed[1], evidence(call.line));
984 table.indexes.push(DatabaseIndex {
985 name: Some(bounded_identifier("ed[0])),
986 columns,
987 unique,
988 evidence: evidence(call.line),
989 });
990 }
991 document
992}
993
994fn parse_sqlalchemy(source_path: &str, input: &str) -> DataDocument {
995 let mut document = empty_document(source_path, DataArtifactKind::SqlAlchemy);
996 for class in python_classes(input) {
997 let Some(table_name) = assignment_literal(class.body, "__tablename__") else {
998 continue;
999 };
1000 let mut table = table_from_qualified_text(&table_name, evidence(class.line));
1001 table.schema = table_argument_literal(class.body, "schema");
1002 for (offset, line) in class.body.lines().enumerate() {
1003 let Some(column_position) = line.find("Column(") else {
1004 continue;
1005 };
1006 let Some((attribute, _)) = line[..column_position].split_once('=') else {
1007 continue;
1008 };
1009 let open = column_position + "Column(".len();
1010 let body = balanced_slice(&line[open..]).unwrap_or(&line[open..]);
1011 if let Some(column) = python_column_with_fallback(
1012 body,
1013 evidence(add_line(class.line, offset)),
1014 Some(attribute.trim()),
1015 ) {
1016 if let Some(reference) = call_literal(body, "ForeignKey") {
1017 let (foreign_table, foreign_column) = split_reference(&reference);
1018 table.foreign_keys.push(DatabaseForeignKey {
1019 name: None,
1020 columns: vec![column.name.clone()],
1021 referenced_table: foreign_table,
1022 referenced_columns: foreign_column.into_iter().collect(),
1023 evidence: column.evidence,
1024 });
1025 }
1026 table.columns.push(column);
1027 }
1028 }
1029 document.owners.push(class.name.to_owned());
1030 document.accesses.push(DataAccessObservation {
1031 role: DataAccessRole::ModelBinding,
1032 table: qualified_table_name(&table),
1033 model: None,
1034 owner: Some(class.name.to_owned()),
1035 operation: None,
1036 evidence: table.evidence,
1037 });
1038 document.tables.push(table);
1039 }
1040 document
1041}
1042
1043fn parse_diesel(source_path: &str, input: &str) -> DataDocument {
1044 let mut document = empty_document(source_path, DataArtifactKind::Diesel);
1045 for block in macro_blocks(input, "table!") {
1046 let header = block
1047 .body
1048 .lines()
1049 .find(|line| line.contains('('))
1050 .unwrap_or_default()
1051 .trim();
1052 let name = header.split('(').next().unwrap_or_default().trim();
1053 if name.is_empty() {
1054 mark_incomplete(&mut document, DataWarning::UnsupportedConstruct);
1055 continue;
1056 }
1057 let mut table = table_from_qualified_text(name, evidence(block.line));
1058 let primary_keys = header
1059 .split_once('(')
1060 .and_then(|(_, rest)| rest.split_once(')'))
1061 .map_or_else(Vec::new, |(keys, _)| comma_identifiers(keys));
1062 for (offset, raw_line) in block.body.lines().enumerate() {
1063 let Some((name, data_type)) = raw_line.split_once("->") else {
1064 continue;
1065 };
1066 let name = bounded_identifier(name.trim());
1067 if name.is_empty() {
1068 continue;
1069 }
1070 table.columns.push(DatabaseColumn {
1071 primary_key: primary_keys.contains(&name),
1072 name,
1073 data_type: Some(bounded_identifier(
1074 data_type.trim().trim_end_matches(',').trim(),
1075 )),
1076 nullable: Some(data_type.contains("Nullable<")),
1077 unique: false,
1078 default_present: false,
1079 evidence: evidence(add_line(block.line, offset)),
1080 });
1081 }
1082 push_declaration_access(&mut document, &table, None);
1083 document.tables.push(table);
1084 }
1085 document
1086}
1087
1088fn append_statement_accesses(
1089 statement: &Statement,
1090 line: DataEvidenceLine,
1091 owner: Option<&str>,
1092 document: &mut DataDocument,
1093 depth: usize,
1094) {
1095 if depth >= MAX_DATA_DEPTH {
1096 mark_incomplete(document, DataWarning::LimitExceeded);
1097 return;
1098 }
1099 let mut observations = BTreeSet::new();
1100 match statement {
1101 Statement::Query(query) => {
1102 let mut tables = BTreeSet::new();
1103 collect_query_tables(query, &mut tables, depth + 1);
1104 for table in tables {
1105 observations.insert((DataAccessRole::Reader, DataOperation::Select, table));
1106 }
1107 }
1108 Statement::Insert(insert) => {
1109 if let TableObject::TableName(name) = &insert.table {
1110 observations.insert((
1111 DataAccessRole::Writer,
1112 DataOperation::Insert,
1113 object_name(name),
1114 ));
1115 }
1116 if let Some(query) = insert.source.as_deref() {
1117 let mut tables = BTreeSet::new();
1118 collect_query_tables(query, &mut tables, depth + 1);
1119 for table in tables {
1120 observations.insert((DataAccessRole::Reader, DataOperation::Select, table));
1121 }
1122 }
1123 }
1124 Statement::Update(update) => {
1125 let mut tables = BTreeSet::new();
1126 collect_table_with_joins(&update.table, &mut tables, depth + 1);
1127 if let Some(table) = tables.into_iter().next() {
1128 observations.insert((DataAccessRole::Writer, DataOperation::Update, table));
1129 }
1130 }
1131 Statement::Delete(delete) => {
1132 let tables = match &delete.from {
1133 FromTable::WithFromKeyword(tables) | FromTable::WithoutKeyword(tables) => tables,
1134 };
1135 let mut names = BTreeSet::new();
1136 for table in tables {
1137 collect_table_with_joins(table, &mut names, depth + 1);
1138 }
1139 for table in names {
1140 observations.insert((DataAccessRole::Writer, DataOperation::Delete, table));
1141 }
1142 }
1143 _ => mark_incomplete(document, DataWarning::UnsupportedConstruct),
1144 }
1145 document
1146 .accesses
1147 .extend(
1148 observations
1149 .into_iter()
1150 .map(|(role, operation, table)| DataAccessObservation {
1151 role,
1152 table,
1153 model: None,
1154 owner: owner.map(ToOwned::to_owned),
1155 operation: Some(operation),
1156 evidence: line,
1157 }),
1158 );
1159}
1160
1161fn collect_query_tables(query: &Query, output: &mut BTreeSet<String>, depth: usize) {
1162 if depth >= MAX_DATA_DEPTH {
1163 return;
1164 }
1165 if let Some(with) = &query.with {
1166 for cte in &with.cte_tables {
1167 collect_query_tables(&cte.query, output, depth + 1);
1168 }
1169 }
1170 collect_set_expr_tables(&query.body, output, depth + 1);
1171}
1172
1173fn collect_set_expr_tables(expression: &SetExpr, output: &mut BTreeSet<String>, depth: usize) {
1174 if depth >= MAX_DATA_DEPTH {
1175 return;
1176 }
1177 match expression {
1178 SetExpr::Select(select) => {
1179 for table in &select.from {
1180 collect_table_with_joins(table, output, depth + 1);
1181 }
1182 }
1183 SetExpr::Query(query) => collect_query_tables(query, output, depth + 1),
1184 SetExpr::SetOperation { left, right, .. } => {
1185 collect_set_expr_tables(left, output, depth + 1);
1186 collect_set_expr_tables(right, output, depth + 1);
1187 }
1188 SetExpr::Insert(statement) | SetExpr::Update(statement) | SetExpr::Delete(statement) => {
1189 let mut ignored = empty_document("", DataArtifactKind::LiteralQuerySource);
1190 append_statement_accesses(statement, evidence(1), None, &mut ignored, depth + 1);
1191 output.extend(ignored.accesses.into_iter().map(|access| access.table));
1192 }
1193 SetExpr::Table(table) => {
1194 if let Some(table_name) = table.table_name.as_deref() {
1195 output.insert(bounded_identifier(table_name));
1196 }
1197 }
1198 SetExpr::Values(_) | SetExpr::Merge(_) => {}
1199 }
1200}
1201
1202fn collect_table_with_joins(table: &TableWithJoins, output: &mut BTreeSet<String>, depth: usize) {
1203 collect_table_factor(&table.relation, output, depth + 1);
1204 for join in &table.joins {
1205 collect_table_factor(&join.relation, output, depth + 1);
1206 }
1207}
1208
1209fn collect_table_factor(factor: &TableFactor, output: &mut BTreeSet<String>, depth: usize) {
1210 if depth >= MAX_DATA_DEPTH {
1211 return;
1212 }
1213 match factor {
1214 TableFactor::Table { name, args, .. } if args.is_none() => {
1215 output.insert(object_name(name));
1216 }
1217 TableFactor::Derived { subquery, .. } => collect_query_tables(subquery, output, depth + 1),
1218 _ => {}
1219 }
1220}
1221
1222fn table_from_name(name: &ObjectName, line: DataEvidenceLine) -> DatabaseTable {
1223 table_from_qualified_text(&object_name(name), line)
1224}
1225
1226fn table_from_qualified_text(name: &str, line: DataEvidenceLine) -> DatabaseTable {
1227 let parts = name
1228 .split('.')
1229 .map(clean_identifier)
1230 .filter(|part| !part.is_empty())
1231 .collect::<Vec<_>>();
1232 let table_name = parts.last().cloned().unwrap_or_default();
1233 let schema = (parts.len() >= 2).then(|| parts[parts.len() - 2].clone());
1234 let database = (parts.len() >= 3).then(|| parts[parts.len() - 3].clone());
1235 DatabaseTable {
1236 database,
1237 schema,
1238 name: table_name,
1239 columns: Vec::new(),
1240 indexes: Vec::new(),
1241 foreign_keys: Vec::new(),
1242 evidence: line,
1243 }
1244}
1245
1246fn ensure_table<'a>(
1247 document: &'a mut DataDocument,
1248 name: &ObjectName,
1249 line: DataEvidenceLine,
1250) -> &'a mut DatabaseTable {
1251 ensure_text_table(document, &object_name(name), line)
1252}
1253
1254fn ensure_text_table<'a>(
1255 document: &'a mut DataDocument,
1256 name: &str,
1257 line: DataEvidenceLine,
1258) -> &'a mut DatabaseTable {
1259 let candidate = table_from_qualified_text(name, line);
1260 let key = table_key(&candidate);
1261 if let Some(index) = document
1262 .tables
1263 .iter()
1264 .position(|table| table_key(table) == key)
1265 {
1266 return &mut document.tables[index];
1267 }
1268 document.tables.push(candidate);
1269 let index = document.tables.len() - 1;
1270 &mut document.tables[index]
1271}
1272
1273fn push_declaration_access(
1274 document: &mut DataDocument,
1275 table: &DatabaseTable,
1276 owner: Option<&str>,
1277) {
1278 document.accesses.push(DataAccessObservation {
1279 role: DataAccessRole::Declaration,
1280 table: qualified_table_name(table),
1281 model: None,
1282 owner: owner.map(ToOwned::to_owned),
1283 operation: None,
1284 evidence: table.evidence,
1285 });
1286}
1287
1288fn mark_columns(table: &mut DatabaseTable, names: &[String], primary: bool, unique: bool) {
1289 for column in &mut table.columns {
1290 if names
1291 .iter()
1292 .any(|name| clean_identifier(name) == column.name)
1293 {
1294 column.primary_key |= primary;
1295 column.unique |= unique;
1296 }
1297 }
1298}
1299
1300fn object_name(name: &ObjectName) -> String {
1301 bounded_identifier(&name.to_string())
1302}
1303
1304fn index_column_name(column: &IndexColumn) -> Option<String> {
1305 match &column.column.expr {
1306 Expr::Identifier(identifier) => Some(bounded_identifier(&identifier.value)),
1307 Expr::CompoundIdentifier(identifiers) => Some(
1308 identifiers
1309 .iter()
1310 .map(|identifier| bounded_identifier(&identifier.value))
1311 .collect::<Vec<_>>()
1312 .join("."),
1313 ),
1314 _ => None,
1315 }
1316}
1317
1318fn safe_data_type(value: &str) -> String {
1319 if value.contains(['\'', '"']) {
1320 value.split_once('(').map_or_else(
1321 || bounded_identifier(value),
1322 |(name, _)| bounded_identifier(name),
1323 )
1324 } else {
1325 bounded_identifier(value)
1326 }
1327}
1328
1329fn qualified_table_name(table: &DatabaseTable) -> String {
1330 [
1331 table.database.as_deref(),
1332 table.schema.as_deref(),
1333 Some(&table.name),
1334 ]
1335 .into_iter()
1336 .flatten()
1337 .collect::<Vec<_>>()
1338 .join(".")
1339}
1340
1341fn table_key(table: &DatabaseTable) -> (Option<&str>, Option<&str>, &str) {
1342 (
1343 table.database.as_deref(),
1344 table.schema.as_deref(),
1345 &table.name,
1346 )
1347}
1348
1349fn finish_document(document: &mut DataDocument) {
1350 let mut tables = BTreeMap::<(Option<String>, Option<String>, String), DatabaseTable>::new();
1351 for table in std::mem::take(&mut document.tables) {
1352 let key = (
1353 table.database.clone(),
1354 table.schema.clone(),
1355 table.name.clone(),
1356 );
1357 if let Some(existing) = tables.get_mut(&key) {
1358 existing.columns.extend(table.columns);
1359 existing.indexes.extend(table.indexes);
1360 existing.foreign_keys.extend(table.foreign_keys);
1361 existing.evidence = existing.evidence.min(table.evidence);
1362 } else {
1363 tables.insert(key, table);
1364 }
1365 }
1366 document.tables = tables.into_values().collect();
1367 for table in &mut document.tables {
1368 table.columns.sort();
1369 table.columns.dedup();
1370 table.indexes.sort();
1371 table.indexes.dedup();
1372 table.foreign_keys.sort();
1373 table.foreign_keys.dedup();
1374 }
1375 document.accesses.sort();
1376 document.accesses.dedup();
1377 document.frameworks.sort();
1378 document.frameworks.dedup();
1379 document.references.sort();
1380 document.references.dedup();
1381 document.owners.sort();
1382 document.owners.dedup();
1383 document.warnings.sort();
1384 document.warnings.dedup();
1385 document.incomplete |= !document.warnings.is_empty();
1386
1387 if structured_item_count(document) > MAX_DATA_ITEMS {
1388 truncate_structured_items(document);
1389 mark_incomplete(document, DataWarning::LimitExceeded);
1390 document.warnings.sort();
1391 document.warnings.dedup();
1392 }
1393
1394 let databases = document
1395 .tables
1396 .iter()
1397 .filter_map(|table| table.database.clone())
1398 .collect::<BTreeSet<_>>();
1399 let schemas = document
1400 .tables
1401 .iter()
1402 .filter_map(|table| table.schema.clone())
1403 .collect::<BTreeSet<_>>();
1404 document.database_name = single_value(databases);
1405 document.schema_name = single_value(schemas);
1406}
1407
1408fn structured_item_count(document: &DataDocument) -> usize {
1409 document.tables.len()
1410 + document.accesses.len()
1411 + document.frameworks.len()
1412 + document.references.len()
1413 + document.owners.len()
1414 + document
1415 .tables
1416 .iter()
1417 .map(|table| table.columns.len() + table.indexes.len() + table.foreign_keys.len())
1418 .sum::<usize>()
1419}
1420
1421fn truncate_structured_items(document: &mut DataDocument) {
1422 let mut remaining = MAX_DATA_ITEMS;
1423
1424 document.tables.truncate(remaining);
1425 remaining = remaining.saturating_sub(document.tables.len());
1426 document.accesses.truncate(remaining);
1427 remaining = remaining.saturating_sub(document.accesses.len());
1428 document.frameworks.truncate(remaining);
1429 remaining = remaining.saturating_sub(document.frameworks.len());
1430 document.references.truncate(remaining);
1431 remaining = remaining.saturating_sub(document.references.len());
1432 document.owners.truncate(remaining);
1433 remaining = remaining.saturating_sub(document.owners.len());
1434
1435 for table in &mut document.tables {
1436 table.columns.truncate(remaining);
1437 remaining = remaining.saturating_sub(table.columns.len());
1438 table.indexes.truncate(remaining);
1439 remaining = remaining.saturating_sub(table.indexes.len());
1440 table.foreign_keys.truncate(remaining);
1441 remaining = remaining.saturating_sub(table.foreign_keys.len());
1442 }
1443}
1444
1445fn single_value(values: BTreeSet<String>) -> Option<String> {
1446 (values.len() == 1)
1447 .then(|| values.into_iter().next())
1448 .flatten()
1449}
1450
1451fn mark_incomplete(document: &mut DataDocument, warning: DataWarning) {
1452 document.incomplete = true;
1453 document.warnings.push(warning);
1454}
1455
1456fn evidence(line: u32) -> DataEvidenceLine {
1457 DataEvidenceLine(line.clamp(1, MAX_DATA_SOURCE_LINES))
1458}
1459
1460fn add_evidence(base: u32, relative: u32) -> DataEvidenceLine {
1461 evidence(base.saturating_add(relative.saturating_sub(1)))
1462}
1463
1464fn add_line(base: u32, offset: usize) -> u32 {
1465 base.saturating_add(u32::try_from(offset).unwrap_or(u32::MAX))
1466 .min(MAX_DATA_SOURCE_LINES)
1467}
1468
1469fn bounded_identifier(value: &str) -> String {
1470 value.trim().chars().take(MAX_IDENTIFIER_CHARS).collect()
1471}
1472
1473fn clean_identifier(value: &str) -> String {
1474 bounded_identifier(
1475 value.trim_matches(|character| matches!(character, '"' | '\'' | '`' | '[' | ']')),
1476 )
1477}
1478
1479fn filename_order_hint(source_path: &str) -> Option<u64> {
1480 Path::new(source_path)
1481 .file_name()
1482 .and_then(|name| name.to_str())
1483 .and_then(|name| {
1484 let digits = name
1485 .chars()
1486 .take_while(char::is_ascii_digit)
1487 .collect::<String>();
1488 (!digits.is_empty()).then_some(digits)
1489 })
1490 .and_then(|digits| digits.parse().ok())
1491}
1492
1493fn sql_migration_path(normalized_path: &str, filename: &str) -> bool {
1494 normalized_path
1495 .split('/')
1496 .any(|component| component == "migrations")
1497 || filename.ends_with(".up.sql")
1498 || filename.ends_with(".down.sql")
1499}
1500
1501fn sql_contains_only_queries(input: &str) -> bool {
1502 Parser::parse_sql(&GenericDialect {}, input).is_ok_and(|statements| {
1503 !statements.is_empty()
1504 && statements.iter().all(|statement| {
1505 matches!(
1506 statement,
1507 Statement::Query(_)
1508 | Statement::Insert(_)
1509 | Statement::Update(_)
1510 | Statement::Delete(_)
1511 )
1512 })
1513 })
1514}
1515
1516fn sql_migration_metadata(source_path: &str, input: &str) -> MigrationMetadata {
1517 let filename = Path::new(source_path)
1518 .file_name()
1519 .and_then(|name| name.to_str())
1520 .unwrap_or_default();
1521 let order_hint = filename_order_hint(source_path);
1522 let revision = order_hint.map(|value| value.to_string());
1523 MigrationMetadata {
1524 revision,
1525 down_revision: None,
1526 order_hint,
1527 reversible: filename.ends_with(".up.sql")
1528 || filename.ends_with(".down.sql")
1529 || contains_keyword(input, "ROLLBACK"),
1530 evidence: evidence(1),
1531 }
1532}
1533
1534fn inferred_crate_root(source_path: &str) -> String {
1535 let normalized = source_path.replace('\\', "/");
1536 for marker in ["/src/", "/tests/", "/examples/", "/benches/"] {
1537 if let Some((root, _)) = normalized.split_once(marker) {
1538 return root.to_owned();
1539 }
1540 }
1541 if normalized.starts_with("src/")
1542 || normalized.starts_with("tests/")
1543 || normalized.starts_with("examples/")
1544 || normalized.starts_with("benches/")
1545 {
1546 return String::new();
1547 }
1548 normalized
1549 .rsplit_once('/')
1550 .map_or_else(String::new, |(parent, _)| parent.to_owned())
1551}
1552
1553fn normalize_data_reference(crate_root: &str, reference: &str) -> Option<String> {
1554 let reference = reference.replace('\\', "/");
1555 if reference.is_empty()
1556 || reference.starts_with('/')
1557 || reference.as_bytes().get(1) == Some(&b':')
1558 {
1559 return None;
1560 }
1561 let mut components = crate_root
1562 .replace('\\', "/")
1563 .split('/')
1564 .filter(|component| !component.is_empty() && *component != ".")
1565 .map(str::to_owned)
1566 .collect::<Vec<_>>();
1567 for component in reference.split('/') {
1568 match component {
1569 "" | "." => {}
1570 ".." if !components.is_empty() => {
1571 components.pop();
1572 }
1573 ".." => return None,
1574 value => components.push(value.to_owned()),
1575 }
1576 }
1577 (!components.is_empty()).then(|| components.join("/"))
1578}
1579
1580fn sql_statement_lines(input: &str) -> Vec<u32> {
1581 let mut lines = Vec::new();
1582 let mut line = 1_u32;
1583 let mut quote = None;
1584 let mut escaped = false;
1585 let mut statement_started = false;
1586 let mut line_comment = false;
1587 let mut block_comment = false;
1588 let mut characters = input.chars().peekable();
1589 while let Some(character) = characters.next() {
1590 if character == '\n' {
1591 line = line.saturating_add(1);
1592 line_comment = false;
1593 continue;
1594 }
1595 if line_comment {
1596 continue;
1597 }
1598 if block_comment {
1599 if character == '*' && characters.peek() == Some(&'/') {
1600 characters.next();
1601 block_comment = false;
1602 }
1603 continue;
1604 }
1605 if escaped {
1606 escaped = false;
1607 continue;
1608 }
1609 if character == '\\' && quote.is_some() {
1610 escaped = true;
1611 continue;
1612 }
1613 if let Some(active) = quote {
1614 if character == active {
1615 quote = None;
1616 }
1617 continue;
1618 }
1619 if character == '-' && characters.peek() == Some(&'-') {
1620 characters.next();
1621 line_comment = true;
1622 continue;
1623 }
1624 if character == '/' && characters.peek() == Some(&'*') {
1625 characters.next();
1626 block_comment = true;
1627 continue;
1628 }
1629 if !statement_started && !character.is_whitespace() && character != ';' {
1630 lines.push(line);
1631 statement_started = true;
1632 }
1633 if matches!(character, '\'' | '"' | '`') {
1634 quote = Some(character);
1635 } else if character == ';' {
1636 statement_started = false;
1637 }
1638 }
1639 lines
1640}
1641
1642#[derive(Debug, Clone, Copy)]
1643struct SqlStatementChunk<'a> {
1644 text: &'a str,
1645 line: u32,
1646}
1647
1648fn sql_statement_chunks(input: &str) -> Vec<SqlStatementChunk<'_>> {
1649 let mut output = Vec::new();
1650 let mut start = 0_usize;
1651 let mut line = 1_u32;
1652 let mut statement_line = None;
1653 let mut quote = None;
1654 let mut escaped = false;
1655 let mut line_comment = false;
1656 let mut block_comment = false;
1657 let mut characters = input.char_indices().peekable();
1658 while let Some((index, character)) = characters.next() {
1659 if character == '\n' {
1660 line = line.saturating_add(1);
1661 line_comment = false;
1662 continue;
1663 }
1664 if line_comment {
1665 continue;
1666 }
1667 if block_comment {
1668 if character == '*' && characters.peek().is_some_and(|(_, next)| *next == '/') {
1669 characters.next();
1670 block_comment = false;
1671 }
1672 continue;
1673 }
1674 if escaped {
1675 escaped = false;
1676 continue;
1677 }
1678 if character == '\\' && quote.is_some() {
1679 escaped = true;
1680 continue;
1681 }
1682 if let Some(active) = quote {
1683 if character == active {
1684 quote = None;
1685 }
1686 continue;
1687 }
1688 if character == '-' && characters.peek().is_some_and(|(_, next)| *next == '-') {
1689 characters.next();
1690 line_comment = true;
1691 continue;
1692 }
1693 if character == '#' {
1694 line_comment = true;
1695 continue;
1696 }
1697 if character == '/' && characters.peek().is_some_and(|(_, next)| *next == '*') {
1698 characters.next();
1699 block_comment = true;
1700 continue;
1701 }
1702 if !character.is_whitespace() && character != ';' && statement_line.is_none() {
1703 statement_line = Some(line);
1704 }
1705 if matches!(character, '\'' | '"' | '`') {
1706 quote = Some(character);
1707 } else if character == ';' {
1708 let end = index.saturating_add(character.len_utf8());
1709 if let Some(statement_line) = statement_line.take() {
1710 output.push(SqlStatementChunk {
1711 text: &input[start..end],
1712 line: statement_line,
1713 });
1714 }
1715 start = end;
1716 }
1717 if output.len() >= MAX_DATA_ITEMS {
1718 break;
1719 }
1720 }
1721 if output.len() < MAX_DATA_ITEMS
1722 && let Some(statement_line) = statement_line
1723 && start < input.len()
1724 {
1725 output.push(SqlStatementChunk {
1726 text: &input[start..],
1727 line: statement_line,
1728 });
1729 }
1730 output
1731}
1732
1733fn contains_keyword(input: &str, keyword: &str) -> bool {
1734 input
1735 .split(|character: char| !character.is_ascii_alphanumeric() && character != '_')
1736 .any(|word| word.eq_ignore_ascii_case(keyword))
1737}
1738
1739#[derive(Debug)]
1740struct NamedBlock<'a> {
1741 name: &'a str,
1742 body: &'a str,
1743 line: u32,
1744}
1745
1746fn named_blocks<'a>(input: &'a str, keyword: &str) -> Vec<NamedBlock<'a>> {
1747 let mut output = Vec::new();
1748 let mut offset = 0;
1749 let marker = format!("{keyword} ");
1750 while let Some(relative) = input[offset..].find(&marker) {
1751 let start = offset + relative;
1752 if start > 0
1753 && input[..start]
1754 .chars()
1755 .next_back()
1756 .is_some_and(|character| character.is_ascii_alphanumeric() || character == '_')
1757 {
1758 offset = start + marker.len();
1759 continue;
1760 }
1761 let name_start = start + marker.len();
1762 let name_end = input[name_start..]
1763 .find(|character: char| character.is_whitespace() || character == '{')
1764 .map_or(input.len(), |relative_end| name_start + relative_end);
1765 let Some(open_relative) = input[name_end..].find('{') else {
1766 break;
1767 };
1768 let open = name_end + open_relative;
1769 let Some(close) = matching_delimiter(input, open, '{', '}') else {
1770 break;
1771 };
1772 output.push(NamedBlock {
1773 name: &input[name_start..name_end],
1774 body: &input[open + 1..close],
1775 line: line_at(input, start),
1776 });
1777 if output.len() >= MAX_DATA_ITEMS {
1778 break;
1779 }
1780 offset = close + 1;
1781 }
1782 output
1783}
1784
1785fn macro_blocks<'a>(input: &'a str, macro_name: &str) -> Vec<NamedBlock<'a>> {
1786 let mut output = Vec::new();
1787 let mut offset = 0;
1788 while let Some(relative) = input[offset..].find(macro_name) {
1789 let start = offset + relative;
1790 let Some(open_relative) = input[start + macro_name.len()..].find('{') else {
1791 break;
1792 };
1793 let open = start + macro_name.len() + open_relative;
1794 let Some(close) = matching_delimiter(input, open, '{', '}') else {
1795 break;
1796 };
1797 output.push(NamedBlock {
1798 name: &input[start..start + macro_name.len()],
1799 body: &input[open + 1..close],
1800 line: line_at(input, start),
1801 });
1802 if output.len() >= MAX_DATA_ITEMS {
1803 break;
1804 }
1805 offset = close + 1;
1806 }
1807 output
1808}
1809
1810fn matching_delimiter(input: &str, open: usize, left: char, right: char) -> Option<usize> {
1811 let mut depth = 0_usize;
1812 let mut quote = None;
1813 let mut escaped = false;
1814 for (relative, character) in input[open..].char_indices() {
1815 if escaped {
1816 escaped = false;
1817 continue;
1818 }
1819 if character == '\\' && quote.is_some() {
1820 escaped = true;
1821 continue;
1822 }
1823 if let Some(active) = quote {
1824 if character == active {
1825 quote = None;
1826 }
1827 continue;
1828 }
1829 if matches!(character, '\'' | '"' | '`') {
1830 quote = Some(character);
1831 } else if character == left {
1832 depth += 1;
1833 if depth > MAX_DATA_DEPTH {
1834 return None;
1835 }
1836 } else if character == right {
1837 depth = depth.saturating_sub(1);
1838 if depth == 0 {
1839 return Some(open + relative);
1840 }
1841 }
1842 }
1843 None
1844}
1845
1846fn line_at(input: &str, byte: usize) -> u32 {
1847 u32::try_from(
1848 input[..byte.min(input.len())]
1849 .bytes()
1850 .filter(|byte| *byte == b'\n')
1851 .count()
1852 + 1,
1853 )
1854 .unwrap_or(MAX_DATA_SOURCE_LINES)
1855 .min(MAX_DATA_SOURCE_LINES)
1856}
1857
1858fn find_call_literal(input: &str, marker: &str) -> Option<String> {
1859 let position = input.find(marker)?;
1860 first_quoted(&input[position + marker.len()..])
1861}
1862
1863fn bracket_values_after(input: &str, marker: &str) -> Option<Vec<String>> {
1864 let position = input.find(marker)?;
1865 bracket_values(&input[position + marker.len()..])
1866}
1867
1868fn bracket_values(input: &str) -> Option<Vec<String>> {
1869 let open = input.find('[')?;
1870 let close = matching_delimiter(input, open, '[', ']')?;
1871 Some(comma_identifiers(&input[open + 1..close]))
1872}
1873
1874fn comma_identifiers(input: &str) -> Vec<String> {
1875 input
1876 .split(',')
1877 .map(clean_identifier)
1878 .filter(|value| !value.is_empty())
1879 .collect()
1880}
1881
1882#[derive(Debug)]
1883struct Call {
1884 body: String,
1885 line: u32,
1886}
1887
1888fn collect_calls(input: &str, marker: &str) -> Vec<Call> {
1889 let mut output = Vec::new();
1890 let mut offset = 0;
1891 while let Some(relative) = input[offset..].find(marker) {
1892 let start = offset + relative;
1893 let body_start = start + marker.len();
1894 let Some(body) = balanced_slice(&input[body_start..]) else {
1895 break;
1896 };
1897 output.push(Call {
1898 body: body.to_owned(),
1899 line: line_at(input, start),
1900 });
1901 offset = body_start + body.len() + 1;
1902 if output.len() >= MAX_DATA_ITEMS {
1903 break;
1904 }
1905 }
1906 output
1907}
1908
1909fn balanced_slice(input: &str) -> Option<&str> {
1910 let mut depth = 1_usize;
1911 let mut quote = None;
1912 let mut escaped = false;
1913 for (index, character) in input.char_indices() {
1914 if escaped {
1915 escaped = false;
1916 continue;
1917 }
1918 if character == '\\' && quote.is_some() {
1919 escaped = true;
1920 continue;
1921 }
1922 if let Some(active) = quote {
1923 if character == active {
1924 quote = None;
1925 }
1926 continue;
1927 }
1928 if matches!(character, '\'' | '"' | '`') {
1929 quote = Some(character);
1930 } else if character == '(' {
1931 depth += 1;
1932 if depth > MAX_DATA_DEPTH {
1933 return None;
1934 }
1935 } else if character == ')' {
1936 depth -= 1;
1937 if depth == 0 {
1938 return Some(&input[..index]);
1939 }
1940 }
1941 }
1942 None
1943}
1944
1945fn first_quoted(input: &str) -> Option<String> {
1946 quoted_values(input).into_iter().next()
1947}
1948
1949fn quoted_values(input: &str) -> Vec<String> {
1950 let mut output = Vec::new();
1951 let bytes = input.as_bytes();
1952 let mut index = 0;
1953 while index < bytes.len() {
1954 if !matches!(bytes[index], b'\'' | b'"') {
1955 index += 1;
1956 continue;
1957 }
1958 let quote = bytes[index];
1959 let start = index + 1;
1960 index = start;
1961 let mut escaped = false;
1962 while index < bytes.len() {
1963 if escaped {
1964 escaped = false;
1965 } else if bytes[index] == b'\\' {
1966 escaped = true;
1967 } else if bytes[index] == quote {
1968 output.push(bounded_identifier(&input[start..index]));
1969 index += 1;
1970 break;
1971 }
1972 index += 1;
1973 }
1974 }
1975 output
1976}
1977
1978fn assignment_literal(input: &str, name: &str) -> Option<String> {
1979 input.lines().find_map(|line| {
1980 let (left, right) = line.split_once('=')?;
1981 (left.trim() == name).then(|| first_quoted(right)).flatten()
1982 })
1983}
1984
1985fn assignment_line(input: &str, name: &str) -> Option<u32> {
1986 input.lines().enumerate().find_map(|(index, line)| {
1987 let (left, _) = line.split_once('=')?;
1988 (left.trim() == name).then(|| u32::try_from(index + 1).unwrap_or(MAX_DATA_SOURCE_LINES))
1989 })
1990}
1991
1992fn python_column(input: &str, line: DataEvidenceLine) -> Option<DatabaseColumn> {
1993 python_column_with_fallback(input, line, None)
1994}
1995
1996fn python_column_with_fallback(
1997 input: &str,
1998 line: DataEvidenceLine,
1999 fallback_name: Option<&str>,
2000) -> Option<DatabaseColumn> {
2001 let values = quoted_values(input);
2002 let explicit_name = input.trim_start().starts_with(['\'', '"']);
2003 let name = if explicit_name {
2004 values.first().cloned()
2005 } else {
2006 fallback_name.map(bounded_identifier)
2007 }?;
2008 let type_position = usize::from(explicit_name);
2009 let data_type = input
2010 .split(',')
2011 .nth(type_position)
2012 .map(str::trim)
2013 .filter(|value| !value.contains("ForeignKey"))
2014 .map(|value| {
2015 bounded_identifier(
2016 value
2017 .trim_start_matches("sa.")
2018 .trim_end_matches("()")
2019 .trim(),
2020 )
2021 });
2022 Some(DatabaseColumn {
2023 name,
2024 data_type,
2025 nullable: if input.contains("nullable=False") {
2026 Some(false)
2027 } else if input.contains("nullable=True") {
2028 Some(true)
2029 } else {
2030 None
2031 },
2032 primary_key: input.contains("primary_key=True"),
2033 unique: input.contains("unique=True"),
2034 default_present: input.contains("default=") || input.contains("server_default="),
2035 evidence: line,
2036 })
2037}
2038
2039fn python_foreign_key(input: &str, line: DataEvidenceLine) -> Option<DatabaseForeignKey> {
2040 let lists = all_bracket_values(input);
2041 let local = lists.first()?.clone();
2042 let references = lists.get(1)?.clone();
2043 let reference = references.first()?;
2044 let (table, column) = split_reference(reference);
2045 Some(DatabaseForeignKey {
2046 name: None,
2047 columns: local,
2048 referenced_table: table,
2049 referenced_columns: column.into_iter().collect(),
2050 evidence: line,
2051 })
2052}
2053
2054fn all_bracket_values(input: &str) -> Vec<Vec<String>> {
2055 let mut output = Vec::new();
2056 let mut offset = 0;
2057 while let Some(relative) = input[offset..].find('[') {
2058 let open = offset + relative;
2059 let Some(close) = matching_delimiter(input, open, '[', ']') else {
2060 break;
2061 };
2062 output.push(
2063 quoted_values(&input[open + 1..close])
2064 .into_iter()
2065 .map(|value| bounded_identifier(&value))
2066 .collect(),
2067 );
2068 offset = close + 1;
2069 }
2070 output
2071}
2072
2073fn split_reference(reference: &str) -> (String, Option<String>) {
2074 reference.rsplit_once('.').map_or_else(
2075 || (bounded_identifier(reference), None),
2076 |(table, column)| (bounded_identifier(table), Some(bounded_identifier(column))),
2077 )
2078}
2079
2080fn table_argument_literal(input: &str, name: &str) -> Option<String> {
2081 input.lines().find_map(|line| {
2082 if !line.contains("__table_args__") || !line.contains(name) {
2083 return None;
2084 }
2085 let position = line.find(name)?;
2086 let remainder = &line[position + name.len()..];
2087 let value_start = remainder.find(':').map_or(0, |colon| colon + 1);
2088 first_quoted(&remainder[value_start..])
2089 })
2090}
2091
2092#[derive(Debug)]
2093struct PythonClass<'a> {
2094 name: &'a str,
2095 body: &'a str,
2096 line: u32,
2097}
2098
2099fn python_classes(input: &str) -> Vec<PythonClass<'_>> {
2100 let mut starts = Vec::new();
2101 let mut byte = 0;
2102 for (index, line) in input.lines().enumerate() {
2103 if let Some(rest) = line.strip_prefix("class ")
2104 && let Some(end) = rest.find(['(', ':'])
2105 {
2106 starts.push((byte, index + 1, &rest[..end]));
2107 if starts.len() >= MAX_DATA_ITEMS {
2108 break;
2109 }
2110 }
2111 byte += line.len() + 1;
2112 }
2113 starts
2114 .iter()
2115 .enumerate()
2116 .map(|(index, (start, line, name))| {
2117 let body_start = input[*start..]
2118 .find('\n')
2119 .map_or(input.len(), |relative| start + relative + 1);
2120 let body_end = starts
2121 .get(index + 1)
2122 .map_or(input.len(), |(next, _, _)| *next);
2123 PythonClass {
2124 name,
2125 body: &input[body_start..body_end],
2126 line: u32::try_from(*line).unwrap_or(MAX_DATA_SOURCE_LINES),
2127 }
2128 })
2129 .collect()
2130}
2131
2132fn call_literal(input: &str, call: &str) -> Option<String> {
2133 let position = input.find(call)?;
2134 first_quoted(&input[position + call.len()..])
2135}
2136
2137#[derive(Debug, Clone)]
2138struct SourceLiteral {
2139 value: String,
2140 line: u32,
2141 start: usize,
2142 end: usize,
2143 dynamic: bool,
2144 interpolated: bool,
2145}
2146
2147fn quoted_literals(input: &str, language: SourceLanguage) -> Vec<SourceLiteral> {
2148 if language == SourceLanguage::Rust {
2149 return rust_source_string_literals(input);
2150 }
2151 let bytes = input.as_bytes();
2152 let mut output = Vec::new();
2153 let mut index = 0;
2154 while index < bytes.len() {
2155 let quote = match bytes[index] {
2156 b'\'' | b'"' | b'`' => bytes[index],
2157 _ => {
2158 index += 1;
2159 continue;
2160 }
2161 };
2162 let triple = language == SourceLanguage::Python
2163 && index + 2 < bytes.len()
2164 && bytes[index + 1] == quote
2165 && bytes[index + 2] == quote;
2166 let delimiter = if triple { 3 } else { 1 };
2167 let content_start = index + delimiter;
2168 let mut cursor = content_start;
2169 let mut escaped = false;
2170 let mut closed = None;
2171 while cursor < bytes.len() {
2172 if triple
2173 && cursor + 2 < bytes.len()
2174 && bytes[cursor] == quote
2175 && bytes[cursor + 1] == quote
2176 && bytes[cursor + 2] == quote
2177 {
2178 closed = Some(cursor);
2179 break;
2180 }
2181 if !triple {
2182 if escaped {
2183 escaped = false;
2184 cursor += 1;
2185 continue;
2186 }
2187 if bytes[cursor] == b'\\' {
2188 escaped = true;
2189 cursor += 1;
2190 continue;
2191 }
2192 if bytes[cursor] == quote {
2193 closed = Some(cursor);
2194 break;
2195 }
2196 }
2197 cursor += 1;
2198 }
2199 let Some(content_end) = closed else {
2200 break;
2201 };
2202 let value = input[content_start..content_end].to_owned();
2203 let end = content_end + delimiter;
2204 let interpolated = is_interpolated(input, language, index, quote, &value);
2205 let dynamic = interpolated || adjacent_dynamic_operator(input, index, end);
2206 output.push(SourceLiteral {
2207 value,
2208 line: line_at(input, index),
2209 start: index,
2210 end,
2211 dynamic,
2212 interpolated,
2213 });
2214 index = end;
2215 if output.len() >= MAX_DATA_ITEMS {
2216 break;
2217 }
2218 }
2219 output
2220}
2221
2222fn rust_source_string_literals(input: &str) -> Vec<SourceLiteral> {
2223 let mut parser = tree_sitter::Parser::new();
2224 let grammar = tree_sitter_rust::LANGUAGE.into();
2225 if parser.set_language(&grammar).is_err() {
2226 return Vec::new();
2227 }
2228 let Some(tree) = parser.parse(input, None) else {
2229 return Vec::new();
2230 };
2231 let mut output = Vec::new();
2232 collect_rust_string_literals(tree.root_node(), input, &mut output);
2233 output.sort_by_key(|literal| literal.start);
2234 output.truncate(MAX_DATA_ITEMS);
2235 output
2236}
2237
2238fn collect_rust_string_literals(
2239 node: SyntaxNode<'_>,
2240 input: &str,
2241 output: &mut Vec<SourceLiteral>,
2242) {
2243 if matches!(
2244 node.kind(),
2245 "string_literal" | "raw_string_literal" | "byte_string_literal" | "raw_byte_string_literal"
2246 ) {
2247 if let Ok(source) = node.utf8_text(input.as_bytes())
2248 && let Some(mut literal) = rust_string_literals(source, 1).into_iter().next()
2249 {
2250 literal.start = literal.start.saturating_add(node.start_byte());
2251 literal.end = literal.end.saturating_add(node.start_byte());
2252 literal.line = u32::try_from(node.start_position().row)
2253 .unwrap_or(MAX_DATA_SOURCE_LINES)
2254 .saturating_add(1);
2255 literal.dynamic = adjacent_dynamic_operator(input, literal.start, literal.end);
2256 output.push(literal);
2257 }
2258 return;
2259 }
2260 let mut cursor = node.walk();
2261 for child in node.children(&mut cursor) {
2262 collect_rust_string_literals(child, input, output);
2263 }
2264}
2265
2266fn rust_string_literals(input: &str, base_line: u32) -> Vec<SourceLiteral> {
2267 let bytes = input.as_bytes();
2268 let mut output = Vec::new();
2269 let mut index = 0;
2270 while index < bytes.len() && output.len() < MAX_DATA_ITEMS {
2271 if bytes[index] == b'r' {
2272 let mut quote = index + 1;
2273 while quote < bytes.len() && bytes[quote] == b'#' {
2274 quote += 1;
2275 }
2276 if quote < bytes.len() && bytes[quote] == b'"' {
2277 let hashes = quote.saturating_sub(index + 1);
2278 let content_start = quote + 1;
2279 let mut cursor = content_start;
2280 while cursor < bytes.len() {
2281 if bytes[cursor] == b'"'
2282 && cursor + hashes < bytes.len()
2283 && (hashes == 0
2284 || bytes[cursor + 1..=cursor + hashes]
2285 .iter()
2286 .all(|byte| *byte == b'#'))
2287 {
2288 let end = cursor + hashes + 1;
2289 output.push(SourceLiteral {
2290 value: input[content_start..cursor].to_owned(),
2291 line: base_line.saturating_add(line_at(input, index).saturating_sub(1)),
2292 start: index,
2293 end,
2294 dynamic: false,
2295 interpolated: false,
2296 });
2297 index = end;
2298 break;
2299 }
2300 cursor += 1;
2301 }
2302 if index >= content_start {
2303 continue;
2304 }
2305 }
2306 }
2307 if bytes[index] != b'"' {
2308 index += 1;
2309 continue;
2310 }
2311 let content_start = index + 1;
2312 let mut cursor = content_start;
2313 let mut escaped = false;
2314 while cursor < bytes.len() {
2315 if escaped {
2316 escaped = false;
2317 cursor += 1;
2318 continue;
2319 }
2320 match bytes[cursor] {
2321 b'\\' => escaped = true,
2322 b'"' => {
2323 let end = cursor + 1;
2324 output.push(SourceLiteral {
2325 value: unescape_rust_string(&input[content_start..cursor]),
2326 line: base_line.saturating_add(line_at(input, index).saturating_sub(1)),
2327 start: index,
2328 end,
2329 dynamic: adjacent_dynamic_operator(input, index, end),
2330 interpolated: false,
2331 });
2332 index = end;
2333 break;
2334 }
2335 _ => {}
2336 }
2337 cursor += 1;
2338 }
2339 if index < content_start {
2340 break;
2341 }
2342 }
2343 output
2344}
2345
2346fn unescape_rust_string(value: &str) -> String {
2347 let mut output = String::with_capacity(value.len());
2348 let mut characters = value.chars();
2349 while let Some(character) = characters.next() {
2350 if character != '\\' {
2351 output.push(character);
2352 continue;
2353 }
2354 match characters.next() {
2355 Some('n') => output.push('\n'),
2356 Some('r') => output.push('\r'),
2357 Some('t') => output.push('\t'),
2358 Some('\\') | None => output.push('\\'),
2359 Some('"') => output.push('"'),
2360 Some('\'') => output.push('\''),
2361 Some(other) => {
2362 output.push('\\');
2363 output.push(other);
2364 }
2365 }
2366 }
2367 output
2368}
2369
2370#[derive(Debug, Default)]
2371struct SqlxImports {
2372 names: BTreeMap<String, String>,
2373}
2374
2375impl SqlxImports {
2376 fn discover(input: &str) -> Self {
2377 let mut imports = Self::default();
2378 for statement in input.split(';') {
2379 let compact = statement
2380 .chars()
2381 .filter(|character| !character.is_whitespace())
2382 .collect::<String>();
2383 let Some(rest) = compact.strip_prefix("usesqlx::") else {
2384 continue;
2385 };
2386 if let Some(group) = rest
2387 .strip_prefix('{')
2388 .and_then(|value| value.strip_suffix('}'))
2389 {
2390 for item in group.split(',') {
2391 imports.insert_item(item);
2392 }
2393 } else {
2394 imports.insert_item(rest);
2395 }
2396 }
2397 imports
2398 }
2399
2400 fn insert_item(&mut self, item: &str) {
2401 let item = item.rsplit("::").next().unwrap_or(item);
2402 if sqlx_api_kind(item).is_some() || matches!(item, "Executor" | "QueryBuilder" | "Migrator")
2403 {
2404 self.names.insert(item.to_owned(), item.to_owned());
2405 return;
2406 }
2407 for (index, _) in item.match_indices("as") {
2408 let (canonical, local_with_as) = item.split_at(index);
2409 let local = &local_with_as[2..];
2410 if !local.is_empty()
2411 && (sqlx_api_kind(canonical).is_some()
2412 || matches!(canonical, "Executor" | "QueryBuilder" | "Migrator"))
2413 {
2414 self.names.insert(local.to_owned(), canonical.to_owned());
2415 return;
2416 }
2417 }
2418 }
2419
2420 fn canonical<'a>(&'a self, local: &'a str) -> Option<&'a str> {
2421 self.names.get(local).map(String::as_str)
2422 }
2423
2424 fn contains(&self, canonical: &str) -> bool {
2425 self.names.values().any(|value| value == canonical)
2426 }
2427}
2428
2429#[derive(Debug, Default)]
2430struct MysqlAsyncImports {
2431 crate_names: BTreeSet<String>,
2432 traits: BTreeMap<String, String>,
2433}
2434
2435impl MysqlAsyncImports {
2436 fn discover(root: SyntaxNode<'_>, input: &str) -> Self {
2437 let mut declarations = Vec::new();
2438 collect_rust_use_declarations(root, input, &mut declarations);
2439 let mut imports = Self {
2440 crate_names: BTreeSet::from(["mysql_async".to_owned()]),
2441 traits: BTreeMap::new(),
2442 };
2443 for declaration in &declarations {
2444 let compact = compact_rust(declaration);
2445 if let Some(alias) = compact
2446 .strip_prefix("usemysql_asyncas")
2447 .and_then(|value| value.strip_suffix(';'))
2448 .filter(|value| is_rust_identifier(value))
2449 {
2450 imports.crate_names.insert(alias.to_owned());
2451 }
2452 }
2453 for declaration in declarations {
2454 imports.insert_declaration(&compact_rust(declaration));
2455 }
2456 imports
2457 }
2458
2459 fn insert_declaration(&mut self, declaration: &str) {
2460 let Some(path) = declaration.strip_prefix("use") else {
2461 return;
2462 };
2463 if !self
2464 .crate_names
2465 .iter()
2466 .any(|name| path.starts_with(&format!("{name}::")))
2467 {
2468 return;
2469 }
2470 if path.contains("prelude::*") {
2471 for name in MYSQL_ASYNC_TRAITS {
2472 self.traits.insert((*name).to_owned(), (*name).to_owned());
2473 }
2474 }
2475 for canonical in MYSQL_ASYNC_TRAITS {
2476 let Some(start) = path.match_indices(canonical).find_map(|(start, _)| {
2477 let before = &path[..start];
2478 let rest = &path[start + canonical.len()..];
2479 let valid_before = before.ends_with([':', '{', ',']);
2480 let valid_after = rest.starts_with("as")
2481 || rest
2482 .chars()
2483 .next()
2484 .is_some_and(|character| matches!(character, ',' | '}' | ';'));
2485 (valid_before && valid_after).then_some(start)
2486 }) else {
2487 continue;
2488 };
2489 let rest = &path[start + canonical.len()..];
2490 let alias = rest
2491 .strip_prefix("as")
2492 .map(|value| {
2493 value
2494 .chars()
2495 .take_while(|character| {
2496 character.is_ascii_alphanumeric() || *character == '_'
2497 })
2498 .collect::<String>()
2499 })
2500 .filter(|value| is_rust_identifier(value))
2501 .unwrap_or_else(|| (*canonical).to_owned());
2502 self.traits.insert(alias, (*canonical).to_owned());
2503 }
2504 }
2505
2506 fn contains(&self, canonical: &str) -> bool {
2507 self.traits.values().any(|value| value == canonical)
2508 }
2509
2510 fn canonical<'a>(&'a self, local: &'a str) -> Option<&'a str> {
2511 self.traits.get(local).map(String::as_str)
2512 }
2513
2514 fn is_qualified(&self, function: &str) -> bool {
2515 self.crate_names
2516 .iter()
2517 .any(|name| function.contains(&format!("{name}::")))
2518 }
2519}
2520
2521const MYSQL_ASYNC_TRAITS: &[&str] = &["Queryable", "Query", "WithParams", "BatchQuery"];
2522
2523fn collect_rust_use_declarations<'a>(
2524 node: SyntaxNode<'a>,
2525 input: &'a str,
2526 output: &mut Vec<&'a str>,
2527) {
2528 if node.kind() == "use_declaration" {
2529 if let Ok(declaration) = node.utf8_text(input.as_bytes()) {
2530 output.push(declaration);
2531 }
2532 return;
2533 }
2534 let mut cursor = node.walk();
2535 for child in node.children(&mut cursor) {
2536 collect_rust_use_declarations(child, input, output);
2537 }
2538}
2539
2540fn compact_rust(input: &str) -> String {
2541 input
2542 .chars()
2543 .filter(|character| !character.is_whitespace())
2544 .collect()
2545}
2546
2547fn is_rust_identifier(value: &str) -> bool {
2548 let mut characters = value.chars();
2549 characters
2550 .next()
2551 .is_some_and(|character| character == '_' || character.is_ascii_alphabetic())
2552 && characters.all(|character| character == '_' || character.is_ascii_alphanumeric())
2553}
2554
2555#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2556enum SqlxApiKind {
2557 Inline,
2558 QueryFile,
2559 MigrationDirectory,
2560 QueryBuilder,
2561}
2562
2563fn sqlx_api_kind(name: &str) -> Option<SqlxApiKind> {
2564 match name {
2565 "query"
2566 | "query_with"
2567 | "query_with_result"
2568 | "query_unchecked"
2569 | "query_as"
2570 | "query_as_with"
2571 | "query_as_with_result"
2572 | "query_as_unchecked"
2573 | "query_scalar"
2574 | "query_scalar_with"
2575 | "query_scalar_with_result"
2576 | "query_scalar_unchecked"
2577 | "raw_sql" => Some(SqlxApiKind::Inline),
2578 "query_file"
2579 | "query_file_unchecked"
2580 | "query_file_as"
2581 | "query_file_as_unchecked"
2582 | "query_file_scalar"
2583 | "query_file_scalar_unchecked" => Some(SqlxApiKind::QueryFile),
2584 "migrate" => Some(SqlxApiKind::MigrationDirectory),
2585 "QueryBuilder" => Some(SqlxApiKind::QueryBuilder),
2586 _ => None,
2587 }
2588}
2589
2590fn extract_rust_database_source(input: &str, crate_root: &str, document: &mut DataDocument) {
2591 let mut parser = tree_sitter::Parser::new();
2592 let grammar = tree_sitter_rust::LANGUAGE.into();
2593 if parser.set_language(&grammar).is_err() {
2594 mark_incomplete(document, DataWarning::UnsupportedConstruct);
2595 return;
2596 }
2597 let Some(tree) = parser.parse(input, None) else {
2598 mark_incomplete(document, DataWarning::UnsupportedConstruct);
2599 return;
2600 };
2601 if tree.root_node().has_error() {
2602 mark_incomplete(document, DataWarning::SqlParseRecovery);
2603 }
2604 let sqlx_imports = SqlxImports::discover(input);
2605 let mysql_async_imports = MysqlAsyncImports::discover(tree.root_node(), input);
2606 visit_rust_data_nodes(
2607 tree.root_node(),
2608 input,
2609 crate_root,
2610 &sqlx_imports,
2611 &mysql_async_imports,
2612 None,
2613 document,
2614 );
2615}
2616
2617fn visit_rust_data_nodes(
2618 node: SyntaxNode<'_>,
2619 input: &str,
2620 crate_root: &str,
2621 sqlx_imports: &SqlxImports,
2622 mysql_async_imports: &MysqlAsyncImports,
2623 owner: Option<&str>,
2624 document: &mut DataDocument,
2625) {
2626 let owned_name = (node.kind() == "function_item")
2627 .then(|| node.child_by_field_name("name"))
2628 .flatten()
2629 .and_then(|name| name.utf8_text(input.as_bytes()).ok())
2630 .map(bounded_identifier);
2631 let owner = owned_name.as_deref().or(owner);
2632
2633 match node.kind() {
2634 "call_expression" => {
2635 if let (Some(function), Some(arguments)) = (
2636 node.child_by_field_name("function"),
2637 node.child_by_field_name("arguments"),
2638 ) && let (Ok(function), Ok(arguments)) = (
2639 function.utf8_text(input.as_bytes()),
2640 arguments.utf8_text(input.as_bytes()),
2641 ) {
2642 inspect_sqlx_call(
2643 function,
2644 arguments,
2645 node.start_position().row,
2646 crate_root,
2647 sqlx_imports,
2648 owner,
2649 document,
2650 );
2651 inspect_mysql_async_call(
2652 function,
2653 arguments,
2654 node.start_position().row,
2655 mysql_async_imports,
2656 owner,
2657 document,
2658 );
2659 }
2660 }
2661 "macro_invocation" => {
2662 if let Ok(invocation) = node.utf8_text(input.as_bytes()) {
2663 inspect_sqlx_macro(
2664 invocation,
2665 node.start_position().row,
2666 crate_root,
2667 sqlx_imports,
2668 owner,
2669 document,
2670 );
2671 }
2672 }
2673 _ => {}
2674 }
2675
2676 let mut cursor = node.walk();
2677 for child in node.children(&mut cursor) {
2678 visit_rust_data_nodes(
2679 child,
2680 input,
2681 crate_root,
2682 sqlx_imports,
2683 mysql_async_imports,
2684 owner,
2685 document,
2686 );
2687 }
2688}
2689
2690fn inspect_sqlx_call(
2691 function: &str,
2692 arguments: &str,
2693 zero_based_line: usize,
2694 crate_root: &str,
2695 imports: &SqlxImports,
2696 owner: Option<&str>,
2697 document: &mut DataDocument,
2698) {
2699 let compact = strip_rust_generics(function);
2700 let local = compact
2701 .rsplit([':', '.'])
2702 .find(|component| !component.is_empty())
2703 .unwrap_or(&compact);
2704 let qualified = compact.contains("sqlx::");
2705 let canonical = if qualified {
2706 local
2707 } else {
2708 imports.canonical(local).unwrap_or(local)
2709 };
2710 let executor_method = matches!(
2711 canonical,
2712 "execute"
2713 | "execute_many"
2714 | "fetch"
2715 | "fetch_many"
2716 | "fetch_all"
2717 | "fetch_one"
2718 | "fetch_optional"
2719 ) && (imports.contains("Executor")
2720 || compact.contains("sqlx::Executor::"));
2721 let kind = sqlx_api_kind(canonical)
2722 .or_else(|| executor_method.then_some(SqlxApiKind::Inline))
2723 .or_else(|| {
2724 let builder = compact.ends_with("QueryBuilder::new")
2725 || compact.ends_with("QueryBuilder::with_arguments")
2726 || imports.canonical(compact.split("::").next().unwrap_or_default())
2727 == Some("QueryBuilder");
2728 builder.then_some(SqlxApiKind::QueryBuilder)
2729 });
2730 let migrator = compact.ends_with("Migrator::new")
2731 && (qualified
2732 || imports.canonical(compact.split("::").next().unwrap_or_default())
2733 == Some("Migrator"));
2734 let kind = if migrator {
2735 Some(SqlxApiKind::MigrationDirectory)
2736 } else if qualified
2737 || executor_method
2738 || imports.canonical(local).is_some()
2739 || kind == Some(SqlxApiKind::QueryBuilder)
2740 {
2741 kind
2742 } else {
2743 None
2744 };
2745 let Some(kind) = kind else {
2746 return;
2747 };
2748 document.frameworks.push(DataFramework::Sqlx);
2749 let base_line = u32::try_from(zero_based_line)
2750 .unwrap_or(MAX_DATA_SOURCE_LINES)
2751 .saturating_add(1);
2752 let literal = rust_string_literals(arguments, base_line)
2753 .into_iter()
2754 .next();
2755 apply_sqlx_api(kind, literal, base_line, crate_root, owner, document);
2756 if kind == SqlxApiKind::QueryBuilder {
2757 mark_incomplete(document, DataWarning::DynamicQuery);
2758 }
2759}
2760
2761fn inspect_mysql_async_call(
2762 function: &str,
2763 arguments: &str,
2764 zero_based_line: usize,
2765 imports: &MysqlAsyncImports,
2766 owner: Option<&str>,
2767 document: &mut DataDocument,
2768) {
2769 let compact = strip_rust_generics(function);
2770 let method = compact
2771 .rsplit([':', '.'])
2772 .find(|component| !component.is_empty())
2773 .unwrap_or(&compact);
2774 let root = compact.split("::").next().unwrap_or_default();
2775 let imported_trait = imports.canonical(root);
2776 let qualified = imports.is_qualified(&compact);
2777 let method_call = compact.contains('.');
2778 let queryable = mysql_async_queryable_method(method)
2779 && (qualified
2780 || imported_trait == Some("Queryable")
2781 || (method_call && imports.contains("Queryable")));
2782 let required_query_trait = mysql_async_query_trait(method);
2783 let fluent = required_query_trait.is_some()
2784 && (qualified
2785 || imported_trait == required_query_trait
2786 || required_query_trait.is_some_and(|required| imports.contains(required)));
2787 if !queryable && !fluent {
2788 return;
2789 }
2790
2791 let base_line = u32::try_from(zero_based_line)
2792 .unwrap_or(MAX_DATA_SOURCE_LINES)
2793 .saturating_add(1);
2794 let literal = if queryable || qualified || imported_trait.is_some() {
2795 rust_string_literals(arguments, base_line)
2796 .into_iter()
2797 .next()
2798 .or_else(|| mysql_async_fluent_literal(function, method, base_line))
2799 } else {
2800 mysql_async_fluent_literal(function, method, base_line)
2801 };
2802 document.frameworks.push(DataFramework::MysqlAsync);
2803 let Some(literal) = literal else {
2804 mark_incomplete(document, DataWarning::DynamicQuery);
2805 return;
2806 };
2807 if literal.dynamic {
2808 mark_incomplete(document, DataWarning::DynamicQuery);
2809 return;
2810 }
2811 append_inline_sql(&literal.value, evidence(literal.line), owner, document);
2812}
2813
2814fn mysql_async_queryable_method(name: &str) -> bool {
2815 matches!(
2816 name,
2817 "query_iter"
2818 | "prep"
2819 | "exec_iter"
2820 | "query"
2821 | "query_first"
2822 | "query_map"
2823 | "query_fold"
2824 | "query_drop"
2825 | "exec_batch"
2826 | "exec"
2827 | "exec_first"
2828 | "exec_map"
2829 | "exec_fold"
2830 | "exec_drop"
2831 | "query_stream"
2832 | "exec_stream"
2833 )
2834}
2835
2836fn mysql_async_query_trait(name: &str) -> Option<&'static str> {
2837 match name {
2838 "run" | "first" | "fetch" | "reduce" | "map" | "stream" | "ignore" => Some("Query"),
2839 "with" => Some("WithParams"),
2840 "batch" => Some("BatchQuery"),
2841 _ => None,
2842 }
2843}
2844
2845fn mysql_async_fluent_literal(
2846 function: &str,
2847 method: &str,
2848 base_line: u32,
2849) -> Option<SourceLiteral> {
2850 let literal = rust_string_literals(function, base_line)
2851 .into_iter()
2852 .next()?;
2853 let suffix = function.get(literal.end..)?;
2854 let suffix = strip_rust_generics(&compact_rust(suffix));
2855 let suffix = suffix.trim_start_matches(')');
2856 let terminal = format!(".{method}");
2857 (suffix == terminal || (suffix.starts_with(".with(") && suffix.ends_with(&terminal)))
2858 .then_some(literal)
2859}
2860
2861fn inspect_sqlx_macro(
2862 invocation: &str,
2863 zero_based_line: usize,
2864 crate_root: &str,
2865 imports: &SqlxImports,
2866 owner: Option<&str>,
2867 document: &mut DataDocument,
2868) {
2869 let Some((name, arguments)) = invocation.split_once('!') else {
2870 return;
2871 };
2872 let compact = name
2873 .chars()
2874 .filter(|character| !character.is_whitespace())
2875 .collect::<String>();
2876 let local = compact.rsplit("::").next().unwrap_or(&compact);
2877 let qualified = compact.contains("sqlx::");
2878 let canonical = if qualified {
2879 local
2880 } else {
2881 imports.canonical(local).unwrap_or(local)
2882 };
2883 let Some(kind) = sqlx_api_kind(canonical) else {
2884 return;
2885 };
2886 if !qualified && imports.canonical(local).is_none() {
2887 return;
2888 }
2889 document.frameworks.push(DataFramework::Sqlx);
2890 let base_line = u32::try_from(zero_based_line)
2891 .unwrap_or(MAX_DATA_SOURCE_LINES)
2892 .saturating_add(1);
2893 let literal = rust_string_literals(arguments, base_line)
2894 .into_iter()
2895 .next();
2896 apply_sqlx_api(kind, literal, base_line, crate_root, owner, document);
2897}
2898
2899fn apply_sqlx_api(
2900 kind: SqlxApiKind,
2901 literal: Option<SourceLiteral>,
2902 call_line: u32,
2903 crate_root: &str,
2904 owner: Option<&str>,
2905 document: &mut DataDocument,
2906) {
2907 match (kind, literal) {
2908 (SqlxApiKind::Inline | SqlxApiKind::QueryBuilder, Some(literal)) if !literal.dynamic => {
2909 append_inline_sql(&literal.value, evidence(literal.line), owner, document);
2910 }
2911 (SqlxApiKind::QueryFile, Some(literal)) if !literal.dynamic => {
2912 push_sqlx_reference(
2913 DataArtifactReferenceKind::QueryFile,
2914 &literal.value,
2915 evidence(literal.line),
2916 crate_root,
2917 owner,
2918 document,
2919 );
2920 }
2921 (SqlxApiKind::MigrationDirectory, literal) => {
2922 let Some(literal) = literal else {
2923 if let Some(owner) = owner {
2924 document.owners.push(owner.to_owned());
2925 }
2926 document.references.push(DataArtifactReference {
2927 framework: DataFramework::Sqlx,
2928 kind: DataArtifactReferenceKind::SqlxDefaultMigrationDirectory,
2929 path: crate_root.replace('\\', "/").trim_matches('/').to_owned(),
2930 owner: owner.map(ToOwned::to_owned),
2931 evidence: evidence(call_line),
2932 });
2933 return;
2934 };
2935 push_sqlx_reference(
2936 DataArtifactReferenceKind::MigrationDirectory,
2937 &literal.value,
2938 evidence(literal.line),
2939 crate_root,
2940 owner,
2941 document,
2942 );
2943 }
2944 _ => mark_incomplete(document, DataWarning::DynamicQuery),
2945 }
2946}
2947
2948fn append_inline_sql(
2949 sql: &str,
2950 line: DataEvidenceLine,
2951 owner: Option<&str>,
2952 document: &mut DataDocument,
2953) {
2954 let Ok(statements) = Parser::parse_sql(&GenericDialect {}, sql) else {
2955 mark_incomplete(document, DataWarning::SqlParseRecovery);
2956 return;
2957 };
2958 if let Some(owner) = owner {
2959 document.owners.push(owner.to_owned());
2960 }
2961 for statement in &statements {
2962 if matches!(
2963 statement,
2964 Statement::Query(_)
2965 | Statement::Insert(_)
2966 | Statement::Update(_)
2967 | Statement::Delete(_)
2968 ) {
2969 append_statement_accesses(statement, line, owner, document, 0);
2970 } else if !append_schema_statement(statement, line, document) {
2971 mark_incomplete(document, DataWarning::UnsupportedConstruct);
2972 }
2973 }
2974}
2975
2976fn push_sqlx_reference(
2977 kind: DataArtifactReferenceKind,
2978 path: &str,
2979 line: DataEvidenceLine,
2980 crate_root: &str,
2981 owner: Option<&str>,
2982 document: &mut DataDocument,
2983) {
2984 let Some(path) = normalize_data_reference(crate_root, path) else {
2985 mark_incomplete(document, DataWarning::UnresolvedReference);
2986 return;
2987 };
2988 if let Some(owner) = owner {
2989 document.owners.push(owner.to_owned());
2990 }
2991 document.references.push(DataArtifactReference {
2992 framework: DataFramework::Sqlx,
2993 kind,
2994 path,
2995 owner: owner.map(ToOwned::to_owned),
2996 evidence: line,
2997 });
2998}
2999
3000fn strip_rust_generics(value: &str) -> String {
3001 let mut output = String::with_capacity(value.len());
3002 let mut depth = 0_u32;
3003 for character in value.chars().filter(|character| !character.is_whitespace()) {
3004 match character {
3005 '<' => depth = depth.saturating_add(1),
3006 '>' => depth = depth.saturating_sub(1),
3007 _ if depth == 0 => output.push(character),
3008 _ => {}
3009 }
3010 }
3011 output.replace("::::", "::")
3012}
3013
3014fn is_interpolated(
3015 input: &str,
3016 language: SourceLanguage,
3017 start: usize,
3018 quote: u8,
3019 value: &str,
3020) -> bool {
3021 (quote == b'`' && value.contains("${"))
3022 || (language == SourceLanguage::Python
3023 && python_string_prefix(input, start).contains(['f', 'F'])
3024 && value.contains('{')
3025 && value.contains('}'))
3026}
3027
3028fn python_string_prefix(input: &str, start: usize) -> &str {
3029 let bytes = input.as_bytes();
3030 let mut prefix_start = start;
3031 while prefix_start > 0
3032 && bytes[prefix_start - 1].is_ascii_alphabetic()
3033 && start.saturating_sub(prefix_start) < 3
3034 {
3035 prefix_start -= 1;
3036 }
3037 &input[prefix_start..start]
3038}
3039
3040fn sanitize_python_f_string(value: &str) -> Option<String> {
3041 let mut output = String::with_capacity(value.len());
3042 let mut characters = value.char_indices().peekable();
3043 while let Some((_, character)) = characters.next() {
3044 if character == '{' {
3045 if characters.peek().is_some_and(|(_, next)| *next == '{') {
3046 characters.next();
3047 output.push('{');
3048 continue;
3049 }
3050 let mut depth = 1_usize;
3051 let mut quote = None;
3052 let mut escaped = false;
3053 for (_, candidate) in characters.by_ref() {
3054 if escaped {
3055 escaped = false;
3056 continue;
3057 }
3058 if candidate == '\\' && quote.is_some() {
3059 escaped = true;
3060 continue;
3061 }
3062 if let Some(active) = quote {
3063 if candidate == active {
3064 quote = None;
3065 }
3066 continue;
3067 }
3068 if matches!(candidate, '\'' | '"') {
3069 quote = Some(candidate);
3070 } else if candidate == '{' {
3071 depth = depth.saturating_add(1);
3072 } else if candidate == '}' {
3073 depth = depth.saturating_sub(1);
3074 if depth == 0 {
3075 break;
3076 }
3077 }
3078 }
3079 if depth != 0 {
3080 return None;
3081 }
3082 output.push_str("__csg_dynamic_value__");
3083 } else if character == '}' && characters.peek().is_some_and(|(_, next)| *next == '}') {
3084 characters.next();
3085 output.push('}');
3086 } else {
3087 output.push(character);
3088 }
3089 }
3090 Some(output)
3091}
3092
3093fn python_format_call_after(input: &str, literal: &SourceLiteral) -> bool {
3094 input
3095 .get(literal.end..)
3096 .is_some_and(|after| after.trim_start().starts_with(".format("))
3097}
3098
3099fn extract_sqlalchemy_accesses(input: &str, document: &mut DataDocument) {
3100 let mut known_models = BTreeSet::new();
3101 for line in input.lines() {
3102 if line.trim_start().starts_with('#') {
3103 continue;
3104 }
3105 if let Some(model) = sqlalchemy_query_model(line) {
3106 known_models.insert(model);
3107 }
3108 for model in sqlalchemy_qualified_constructors(line) {
3109 known_models.insert(model);
3110 }
3111 }
3112 if known_models.is_empty() {
3113 return;
3114 }
3115 document.frameworks.push(DataFramework::SqlAlchemy);
3116 let mut variables = BTreeMap::new();
3117 for (offset, line) in input.lines().enumerate() {
3118 let trimmed = line.trim();
3119 if trimmed.is_empty() || trimmed.starts_with('#') {
3120 continue;
3121 }
3122 let line_number = u32::try_from(offset)
3123 .unwrap_or(MAX_DATA_SOURCE_LINES)
3124 .saturating_add(1);
3125 if let Some((variable, model)) = sqlalchemy_model_assignment(trimmed, &known_models) {
3126 variables.insert(variable, model);
3127 }
3128 if let Some(model) = sqlalchemy_query_model(trimmed) {
3129 append_sqlalchemy_access(
3130 input,
3131 document,
3132 &model,
3133 if trimmed.contains(".update(") || trimmed.contains(".delete(") {
3134 DataAccessRole::Writer
3135 } else {
3136 DataAccessRole::Reader
3137 },
3138 if trimmed.contains(".update(") {
3139 DataOperation::Update
3140 } else if trimmed.contains(".delete(") {
3141 DataOperation::Delete
3142 } else {
3143 DataOperation::Select
3144 },
3145 line_number,
3146 );
3147 }
3148 for (marker, operation) in [
3149 (".add(", DataOperation::Insert),
3150 (".delete(", DataOperation::Delete),
3151 ] {
3152 let Some(argument) = call_argument_identifier(trimmed, marker) else {
3153 continue;
3154 };
3155 let model = variables
3156 .get(&argument)
3157 .cloned()
3158 .or_else(|| known_models.contains(&argument).then_some(argument));
3159 if let Some(model) = model {
3160 append_sqlalchemy_access(
3161 input,
3162 document,
3163 &model,
3164 DataAccessRole::Writer,
3165 operation,
3166 line_number,
3167 );
3168 }
3169 }
3170 }
3171}
3172
3173fn append_sqlalchemy_access(
3174 input: &str,
3175 document: &mut DataDocument,
3176 model: &str,
3177 role: DataAccessRole,
3178 operation: DataOperation,
3179 line: u32,
3180) {
3181 let owner = owner_at_line(SourceLanguage::Python, input, line);
3182 if let Some(owner) = owner.as_ref() {
3183 document.owners.push(owner.clone());
3184 }
3185 document.accesses.push(DataAccessObservation {
3186 role,
3187 table: String::new(),
3188 model: Some(model.to_owned()),
3189 owner,
3190 operation: Some(operation),
3191 evidence: evidence(line),
3192 });
3193}
3194
3195fn sqlalchemy_query_model(line: &str) -> Option<String> {
3196 let (_, after) = line.split_once(".query(")?;
3197 python_model_name(after)
3198}
3199
3200fn sqlalchemy_qualified_constructors(line: &str) -> Vec<String> {
3201 let mut output = Vec::new();
3202 let mut remaining = line;
3203 while let Some((_, after)) = remaining.split_once("models.") {
3204 if let Some(model) = python_model_name(after)
3205 && after[model.len()..].trim_start().starts_with('(')
3206 {
3207 output.push(model);
3208 }
3209 remaining = after.get(1..).unwrap_or_default();
3210 }
3211 output
3212}
3213
3214fn sqlalchemy_model_assignment(
3215 line: &str,
3216 known_models: &BTreeSet<String>,
3217) -> Option<(String, String)> {
3218 let (left, right) = line.split_once('=')?;
3219 let variable = left.trim();
3220 if !is_python_identifier(variable) {
3221 return None;
3222 }
3223 let model = python_model_name(right.trim())?;
3224 known_models
3225 .contains(&model)
3226 .then(|| (variable.to_owned(), model))
3227}
3228
3229fn call_argument_identifier(line: &str, marker: &str) -> Option<String> {
3230 let (_, after) = line.split_once(marker)?;
3231 let argument = after
3232 .trim_start()
3233 .split(|character: char| !character.is_ascii_alphanumeric() && character != '_')
3234 .next()?;
3235 is_python_identifier(argument).then(|| argument.to_owned())
3236}
3237
3238fn python_model_name(input: &str) -> Option<String> {
3239 let qualified = input
3240 .trim_start()
3241 .split(|character: char| {
3242 !character.is_ascii_alphanumeric() && character != '_' && character != '.'
3243 })
3244 .next()?;
3245 let model = qualified.rsplit('.').next()?;
3246 is_python_identifier(model).then(|| model.to_owned())
3247}
3248
3249fn is_python_identifier(value: &str) -> bool {
3250 let mut characters = value.chars();
3251 characters
3252 .next()
3253 .is_some_and(|character| character == '_' || character.is_ascii_alphabetic())
3254 && characters.all(|character| character == '_' || character.is_ascii_alphanumeric())
3255}
3256
3257fn adjacent_dynamic_operator(input: &str, start: usize, end: usize) -> bool {
3258 let before = input[..start].trim_end();
3259 let after = input[end..].trim_start();
3260 before.ends_with('+')
3261 || after.starts_with('+')
3262 || before.ends_with("format!(")
3263 || after.starts_with(".format(")
3264}
3265
3266const QUERY_CONTEXT_MARKERS: [&str; 10] = [
3267 "query", "execute", "fetch", "select", ".sql(", "sql =", "sql!", "sqlx", "prepare", "raw",
3268];
3269
3270static DATA_SOURCE_MARKERS: LazyLock<MarkerSet> = LazyLock::new(|| {
3274 let mut markers = QUERY_CONTEXT_MARKERS.to_vec();
3275 markers.extend(["exec(", "pymysql", "psycopg", "sqlalchemy", "mysql_async"]);
3276 MarkerSet::new(&markers)
3277});
3278
3279fn has_query_context(input: &str, literal: &SourceLiteral) -> bool {
3280 let context_start = floor_char_boundary(input, literal.start.saturating_sub(160));
3281 let before = input[context_start..literal.start].to_ascii_lowercase();
3282 let after_end = floor_char_boundary(input, (literal.end + 80).min(input.len()));
3283 let after = input[literal.end..after_end].to_ascii_lowercase();
3284 QUERY_CONTEXT_MARKERS
3285 .iter()
3286 .any(|marker| before.contains(marker) || after.contains(marker))
3287}
3288
3289fn floor_char_boundary(input: &str, mut index: usize) -> usize {
3290 while index > 0 && !input.is_char_boundary(index) {
3291 index -= 1;
3292 }
3293 index
3294}
3295
3296fn first_sql_keyword(input: &str) -> Option<String> {
3297 strip_leading_sql_comments(input)
3298 .trim_start()
3299 .split(|character: char| !character.is_ascii_alphabetic())
3300 .find(|value| !value.is_empty())
3301 .map(str::to_ascii_uppercase)
3302}
3303
3304fn strip_leading_sql_comments(mut input: &str) -> &str {
3305 loop {
3306 input = input.trim_start();
3307 if let Some(rest) = input.strip_prefix("--").or_else(|| input.strip_prefix('#')) {
3308 input = rest.split_once('\n').map_or("", |(_, remaining)| remaining);
3309 continue;
3310 }
3311 if let Some(rest) = input.strip_prefix("/*") {
3312 input = rest.split_once("*/").map_or("", |(_, remaining)| remaining);
3313 continue;
3314 }
3315 return input;
3316 }
3317}
3318
3319fn source_has_dynamic_query(input: &str, language: SourceLanguage) -> bool {
3320 input.lines().any(|line| {
3321 let lower = line.to_ascii_lowercase();
3322 let query_call = ["query(", "execute(", "queryrow(", "exec(", ".sql(", "raw("]
3323 .iter()
3324 .any(|marker| lower.contains(marker));
3325 if !query_call {
3326 return false;
3327 }
3328 let has_quote = line.contains('"') || line.contains('\'') || line.contains('`');
3329 !has_quote
3330 || line.contains('+')
3331 || line.contains("${")
3332 || (language == SourceLanguage::Python && (line.contains("f\"") || line.contains("f'")))
3333 })
3334}
3335
3336fn owner_at_line(language: SourceLanguage, input: &str, line: u32) -> Option<String> {
3337 let lines = input
3338 .lines()
3339 .take(usize::try_from(line).ok()?)
3340 .collect::<Vec<_>>();
3341 lines.iter().rev().find_map(|line| {
3342 let trimmed = line.trim();
3343 match language {
3344 SourceLanguage::Rust => function_name_after(trimmed, "fn "),
3345 SourceLanguage::Python => function_name_after(trimmed, "def "),
3346 SourceLanguage::JavaScript | SourceLanguage::TypeScript => {
3347 function_name_after(trimmed, "function ").or_else(|| {
3348 trimmed
3349 .split_once('=')
3350 .filter(|(_, right)| right.contains("=>"))
3351 .map(|(left, _)| bounded_identifier(left.trim()))
3352 })
3353 }
3354 SourceLanguage::Go => function_name_after(trimmed, "func "),
3355 SourceLanguage::Java => java_method_name(trimmed),
3356 }
3357 })
3358}
3359
3360fn function_name_after(line: &str, marker: &str) -> Option<String> {
3361 let rest = line.strip_prefix(marker)?;
3362 let name = rest
3363 .split(|character: char| character == '(' || character.is_whitespace())
3364 .next()?;
3365 (!name.is_empty()).then(|| bounded_identifier(name))
3366}
3367
3368fn java_method_name(line: &str) -> Option<String> {
3369 if !line.ends_with('{') || !line.contains('(') {
3370 return None;
3371 }
3372 let before = line.split_once('(')?.0;
3373 before
3374 .split_whitespace()
3375 .next_back()
3376 .map(bounded_identifier)
3377}
3378
3379#[cfg(test)]
3380mod tests {
3381 #[test]
3382 fn source_marker_prefilter_should_not_change_any_corpus_document() {
3383 let corpus = crate::markers::differential_corpus();
3384 assert!(
3385 corpus.len() > 100,
3386 "differential corpus is unexpectedly small"
3387 );
3388 for (language, path, input) in corpus {
3389 assert_eq!(
3390 super::parse_literal_sql_source_with_prefilter(language, &path, ".", &input, true),
3391 super::parse_literal_sql_source_with_prefilter(language, &path, ".", &input, false),
3392 "data prefilter changed {path}"
3393 );
3394 }
3395 }
3396
3397 use super::*;
3398
3399 #[test]
3400 fn sql_schema_extracts_columns_indexes_and_foreign_keys() {
3401 let input = r"
3402 CREATE TABLE public.users (
3403 id BIGINT PRIMARY KEY,
3404 email TEXT NOT NULL UNIQUE,
3405 password_hash TEXT DEFAULT 'secret-default'
3406 );
3407 CREATE TABLE public.orders (
3408 id BIGINT PRIMARY KEY,
3409 user_id BIGINT,
3410 CONSTRAINT fk_user FOREIGN KEY (user_id) REFERENCES public.users(id)
3411 );
3412 CREATE INDEX idx_orders_user ON public.orders(user_id);
3413 ";
3414
3415 let document = extract_data_artifact("db/schema.sql", input).expect("schema should parse");
3416
3417 assert!(
3418 document.tables.iter().any(|table| {
3419 table.name == "orders" && table.indexes.len() == 1 && table.foreign_keys.len() == 1
3420 }),
3421 "orders table should retain index and foreign key"
3422 );
3423 }
3424
3425 #[test]
3426 fn sql_migration_preserves_numeric_order_hint() {
3427 let document = extract_data_artifact(
3428 "migrations/0042_add_users.sql",
3429 "CREATE TABLE users (id INTEGER);",
3430 )
3431 .expect("migration should parse");
3432
3433 assert_eq!(
3434 document
3435 .migration
3436 .and_then(|migration| migration.order_hint),
3437 Some(42)
3438 );
3439 }
3440
3441 #[test]
3442 fn prisma_extracts_mapped_model_and_column() {
3443 let input = r#"
3444 model User {
3445 id Int @id
3446 displayName String @map("display_name")
3447 posts Post[] @relation(fields: [id], references: [userId])
3448 @@map("app_users")
3449 @@schema("tenant")
3450 }
3451 "#;
3452
3453 let document =
3454 extract_data_artifact("prisma/schema.prisma", input).expect("Prisma should parse");
3455
3456 assert!(document.tables.iter().any(|table| {
3457 table.name == "app_users"
3458 && table.schema.as_deref() == Some("tenant")
3459 && table
3460 .columns
3461 .iter()
3462 .any(|column| column.name == "display_name")
3463 }));
3464 }
3465
3466 #[test]
3467 fn alembic_extracts_revision_table_and_added_column() {
3468 let input = r#"
3469revision = "abc123"
3470down_revision = "abc122"
3471def upgrade():
3472 op.create_table(
3473 "users",
3474 sa.Column("id", sa.Integer(), primary_key=True),
3475 )
3476 op.add_column("users", sa.Column("email", sa.String(), nullable=False))
3477def downgrade():
3478 op.drop_table("users")
3479"#;
3480
3481 let document = extract_data_artifact("alembic/versions/0042_users.py", input)
3482 .expect("Alembic should parse");
3483
3484 assert!(document.tables.iter().any(|table| {
3485 table.name == "users" && table.columns.iter().any(|column| column.name == "email")
3486 }));
3487 }
3488
3489 #[test]
3490 fn sqlalchemy_extracts_model_binding_and_foreign_key() {
3491 let input = r#"
3492class Order(Base):
3493 __tablename__ = "orders"
3494 __table_args__ = {"schema": "billing"}
3495 id = Column(Integer, primary_key=True)
3496 user_id = Column(Integer, ForeignKey("public.users.id"), nullable=False)
3497"#;
3498
3499 let document = extract_data_artifact("models.py", input).expect("SQLAlchemy should parse");
3500
3501 assert!(
3502 document.accesses.iter().any(|access| {
3503 access.role == DataAccessRole::ModelBinding
3504 && access.owner.as_deref() == Some("Order")
3505 && access.table == "billing.orders"
3506 }) && document.tables.iter().any(|table| {
3507 table.columns.iter().any(|column| column.name == "id")
3508 && table.foreign_keys.iter().any(|foreign| {
3509 foreign.columns == ["user_id"] && foreign.referenced_table == "public.users"
3510 })
3511 })
3512 );
3513 }
3514
3515 #[test]
3516 fn sqlalchemy_nested_enum_does_not_replace_owning_model() {
3517 let input = r#"
3518class Services(Base):
3519 class EnumServiceType(str, enum.Enum):
3520 PUBLIC = "PUBLIC"
3521
3522 __tablename__ = "services"
3523 id = Column(Integer, primary_key=True)
3524"#;
3525
3526 let document = extract_data_artifact("models.py", input).expect("SQLAlchemy should parse");
3527
3528 assert!(document.accesses.iter().any(|access| {
3529 access.role == DataAccessRole::ModelBinding
3530 && access.owner.as_deref() == Some("Services")
3531 && access.table == "services"
3532 }));
3533 assert!(
3534 !document
3535 .owners
3536 .iter()
3537 .any(|owner| owner == "EnumServiceType")
3538 );
3539 }
3540
3541 #[test]
3542 fn diesel_extracts_table_macro() {
3543 let input = r"
3544diesel::table! {
3545 public.users (id) {
3546 id -> Int8,
3547 email -> Text,
3548 nickname -> Nullable<Text>,
3549 }
3550}
3551";
3552
3553 let document = extract_data_artifact("src/schema.rs", input).expect("Diesel should parse");
3554
3555 assert!(document.tables.iter().any(|table| {
3556 table.name == "users"
3557 && table.schema.as_deref() == Some("public")
3558 && table
3559 .columns
3560 .iter()
3561 .any(|column| column.name == "nickname" && column.nullable == Some(true))
3562 }));
3563 }
3564
3565 #[test]
3566 fn literal_select_records_only_direct_tables() {
3567 let input = r#"
3568fn load(pool: &Pool) {
3569 sqlx::query("SELECT u.id FROM users u JOIN teams t ON t.id = u.team_id");
3570}
3571"#;
3572
3573 let document = parse_literal_sql_source(SourceLanguage::Rust, "src/users.rs", input);
3574
3575 assert_eq!(
3576 document
3577 .accesses
3578 .iter()
3579 .map(|access| (&access.role, access.table.as_str()))
3580 .collect::<Vec<_>>(),
3581 vec![
3582 (&DataAccessRole::Reader, "teams"),
3583 (&DataAccessRole::Reader, "users")
3584 ]
3585 );
3586 }
3587
3588 #[test]
3589 fn sqlx_inline_apis_cover_functions_macros_raw_strings_and_raw_sql() {
3590 let input = r##"
3591use sqlx::Executor;
3592
3593async fn load(pool: &sqlx::PgPool) {
3594 sqlx::query!(r#"SELECT id FROM users WHERE id = $1"#, 7_i64);
3595 sqlx::query_as_unchecked!(User, "UPDATE users SET active = true");
3596 sqlx::query_scalar::<_, i64>("SELECT count(*) FROM teams");
3597 sqlx::raw_sql("DELETE FROM sessions; INSERT INTO audits(id) VALUES (1);");
3598 pool.fetch_one("SELECT id FROM accounts");
3599}
3600"##;
3601
3602 let document = parse_literal_sql_source(SourceLanguage::Rust, "src/users.rs", input);
3603 let facts = document
3604 .accesses
3605 .iter()
3606 .map(|access| (access.role, access.operation, access.table.as_str()))
3607 .collect::<BTreeSet<_>>();
3608
3609 assert!(document.frameworks.contains(&DataFramework::Sqlx));
3610 assert!(facts.contains(&(DataAccessRole::Reader, Some(DataOperation::Select), "users")));
3611 assert!(facts.contains(&(DataAccessRole::Writer, Some(DataOperation::Update), "users")));
3612 assert!(facts.contains(&(DataAccessRole::Reader, Some(DataOperation::Select), "teams")));
3613 assert!(facts.contains(&(
3614 DataAccessRole::Writer,
3615 Some(DataOperation::Delete),
3616 "sessions"
3617 )));
3618 assert!(facts.contains(&(
3619 DataAccessRole::Writer,
3620 Some(DataOperation::Insert),
3621 "audits"
3622 )));
3623 assert!(facts.contains(&(
3624 DataAccessRole::Reader,
3625 Some(DataOperation::Select),
3626 "accounts"
3627 )));
3628 }
3629
3630 #[test]
3631 fn rust_sqlx_examples_in_comments_and_string_contents_are_not_evidence() {
3632 let input = r##"
3633fn example() {
3634 // sqlx::query!("SELECT id FROM leaked_comment");
3635 let documentation = r#"sqlx::query!("SELECT id FROM leaked_string")"#;
3636}
3637"##;
3638
3639 let document = parse_literal_sql_source(SourceLanguage::Rust, "src/example.rs", input);
3640
3641 assert!(
3642 document.accesses.is_empty()
3643 && document.frameworks.is_empty()
3644 && document.references.is_empty()
3645 );
3646 }
3647
3648 #[test]
3649 fn mysql_async_queryable_methods_cover_text_prepared_and_streaming_apis() {
3650 let input = r##"
3651use mysql_async::prelude::Queryable as DbQueryable;
3652
3653async fn manage(conn: &mut mysql_async::Conn) {
3654 conn.query_iter(r#"SELECT id FROM users"#).await?;
3655 conn.query_first(b"SELECT id FROM teams").await?;
3656 conn.query_map("SELECT id FROM accounts", |row| row).await?;
3657 conn.query_fold("SELECT id FROM audits", (), |_, _| ()).await?;
3658 conn.query_drop("DELETE FROM sessions").await?;
3659 conn.exec_iter("INSERT INTO events(id) VALUES (1)", ()).await?;
3660 conn.exec_batch("UPDATE jobs SET active = true", [()]).await?;
3661 DbQueryable::exec_drop(conn, "DELETE FROM tokens", ()).await?;
3662 conn.query_stream("SELECT id FROM streams").await?;
3663 conn.exec_stream("SELECT id FROM prepared_streams", ()).await?;
3664 conn.prep("UPDATE prepared_jobs SET active = true").await?;
3665}
3666"##;
3667
3668 let document = parse_literal_sql_source(SourceLanguage::Rust, "src/mysql.rs", input);
3669 let facts = document
3670 .accesses
3671 .iter()
3672 .map(|access| (access.role, access.operation, access.table.as_str()))
3673 .collect::<BTreeSet<_>>();
3674
3675 assert_eq!(document.frameworks, vec![DataFramework::MysqlAsync]);
3676 assert!(facts.contains(&(DataAccessRole::Reader, Some(DataOperation::Select), "users")));
3677 assert!(facts.contains(&(DataAccessRole::Reader, Some(DataOperation::Select), "teams")));
3678 assert!(facts.contains(&(
3679 DataAccessRole::Writer,
3680 Some(DataOperation::Delete),
3681 "sessions"
3682 )));
3683 assert!(facts.contains(&(
3684 DataAccessRole::Writer,
3685 Some(DataOperation::Insert),
3686 "events"
3687 )));
3688 assert!(facts.contains(&(DataAccessRole::Writer, Some(DataOperation::Update), "jobs")));
3689 assert!(facts.contains(&(
3690 DataAccessRole::Writer,
3691 Some(DataOperation::Update),
3692 "prepared_jobs"
3693 )));
3694 }
3695
3696 #[test]
3697 fn mysql_async_fluent_query_traits_cover_aliases_params_and_batch() {
3698 let input = r##"
3699use mysql_async::prelude::{
3700 BatchQuery,
3701 Query as DbQuery,
3702 WithParams,
3703};
3704
3705async fn fluent(conn: &mysql_async::Pool) {
3706 "SELECT id FROM users".first(conn).await?;
3707 br#"SELECT id FROM teams"#.fetch(conn).await?;
3708 "UPDATE jobs SET active = true".with(()).ignore(conn).await?;
3709 "INSERT INTO audits(id) VALUES (1)".with([()]).batch(conn).await?;
3710 DbQuery::run("DELETE FROM sessions", conn).await?;
3711}
3712"##;
3713
3714 let document = parse_literal_sql_source(SourceLanguage::Rust, "src/fluent.rs", input);
3715 let facts = document
3716 .accesses
3717 .iter()
3718 .map(|access| (access.role, access.operation, access.table.as_str()))
3719 .collect::<BTreeSet<_>>();
3720
3721 assert!(document.frameworks.contains(&DataFramework::MysqlAsync));
3722 assert_eq!(
3723 facts,
3724 BTreeSet::from([
3725 (DataAccessRole::Reader, Some(DataOperation::Select), "teams"),
3726 (DataAccessRole::Reader, Some(DataOperation::Select), "users"),
3727 (
3728 DataAccessRole::Writer,
3729 Some(DataOperation::Delete),
3730 "sessions"
3731 ),
3732 (
3733 DataAccessRole::Writer,
3734 Some(DataOperation::Insert),
3735 "audits"
3736 ),
3737 (DataAccessRole::Writer, Some(DataOperation::Update), "jobs"),
3738 ])
3739 );
3740 }
3741
3742 #[test]
3743 fn mysql_async_examples_in_comments_and_string_contents_are_not_evidence() {
3744 let input = r##"
3745fn example() {
3746 // use mysql_async::prelude::*;
3747 // conn.query("SELECT id FROM leaked_comment");
3748 let documentation = r#"conn.exec_drop("DELETE FROM leaked_string", ())"#;
3749}
3750"##;
3751
3752 let document = parse_literal_sql_source(SourceLanguage::Rust, "src/example.rs", input);
3753
3754 assert!(document.accesses.is_empty() && document.frameworks.is_empty());
3755 }
3756
3757 #[test]
3758 fn sqlx_query_file_macro_matrix_resolves_from_cargo_crate_root() {
3759 let input = r#"
3760async fn load() {
3761 sqlx::query_file!("queries/one.sql");
3762 sqlx::query_file_unchecked!("queries/two.sql");
3763 sqlx::query_file_as!(User, "queries/three.sql");
3764 sqlx::query_file_as_unchecked!(User, "queries/four.sql");
3765 sqlx::query_file_scalar!("queries/five.sql");
3766 sqlx::query_file_scalar_unchecked!("queries/six.sql");
3767}
3768"#;
3769
3770 let document = parse_literal_sql_source_at_root(
3771 SourceLanguage::Rust,
3772 "crates/api/src/users.rs",
3773 "crates/api",
3774 input,
3775 );
3776
3777 assert_eq!(
3778 document
3779 .references
3780 .iter()
3781 .map(|reference| reference.path.as_str())
3782 .collect::<Vec<_>>(),
3783 vec![
3784 "crates/api/queries/five.sql",
3785 "crates/api/queries/four.sql",
3786 "crates/api/queries/one.sql",
3787 "crates/api/queries/six.sql",
3788 "crates/api/queries/three.sql",
3789 "crates/api/queries/two.sql",
3790 ]
3791 );
3792 assert!(document.references.iter().all(|reference| {
3793 reference.framework == DataFramework::Sqlx
3794 && reference.kind == DataArtifactReferenceKind::QueryFile
3795 && reference.owner.as_deref() == Some("load")
3796 }));
3797 }
3798
3799 #[test]
3800 fn sqlx_migration_apis_resolve_default_custom_and_runtime_directories() {
3801 let input = r#"
3802use sqlx::migrate::Migrator;
3803
3804static DEFAULT: Migrator = sqlx::migrate!();
3805static CUSTOM: Migrator = sqlx::migrate!("db/migrations");
3806
3807async fn runtime() {
3808 Migrator::new(std::path::Path::new("tenant/migrations")).await;
3809}
3810"#;
3811
3812 let document = parse_literal_sql_source_at_root(
3813 SourceLanguage::Rust,
3814 "crates/api/src/lib.rs",
3815 "crates/api",
3816 input,
3817 );
3818
3819 assert_eq!(
3820 document
3821 .references
3822 .iter()
3823 .filter(|reference| {
3824 reference.kind == DataArtifactReferenceKind::MigrationDirectory
3825 })
3826 .map(|reference| reference.path.as_str())
3827 .collect::<Vec<_>>(),
3828 vec!["crates/api/db/migrations", "crates/api/tenant/migrations",]
3829 );
3830 assert!(document.references.iter().any(|reference| {
3831 reference.kind == DataArtifactReferenceKind::SqlxDefaultMigrationDirectory
3832 && reference.path == "crates/api"
3833 }));
3834 }
3835
3836 #[test]
3837 fn sqlx_configuration_extracts_migration_directory_without_leaking_other_values() {
3838 let document = extract_data_artifact(
3839 "crates/api/sqlx.toml",
3840 r#"
3841[database]
3842url = "postgres://secret@example.invalid/private"
3843
3844[migrate]
3845table-name = "app._sqlx_migrations"
3846migrations-dir = "db/migrations" # relative to the crate root
3847
3848[migrate.defaults]
3849migration-type = "reversible"
3850"#,
3851 )
3852 .expect("SQLx configuration should parse");
3853
3854 assert_eq!(document.artifact_kind, DataArtifactKind::SqlxConfiguration);
3855 assert_eq!(document.frameworks, vec![DataFramework::Sqlx]);
3856 assert_eq!(document.references.len(), 1);
3857 assert_eq!(
3858 document.references[0].kind,
3859 DataArtifactReferenceKind::MigrationDirectory
3860 );
3861 assert_eq!(document.references[0].path, "crates/api/db/migrations");
3862 assert!(!format!("{document:?}").contains("secret"));
3863 }
3864
3865 #[test]
3866 fn sqlx_import_alias_and_query_builder_are_recognized_conservatively() {
3867 let input = r#"
3868use sqlx::{query as sql_query, QueryBuilder as SqlBuilder};
3869
3870fn load() {
3871 sql_query("SELECT id FROM users");
3872 let mut builder = SqlBuilder::<sqlx::Postgres>::new("SELECT id FROM teams");
3873 builder.push(" WHERE id = ").push_bind(7_i64);
3874}
3875"#;
3876
3877 let document = parse_literal_sql_source(SourceLanguage::Rust, "src/users.rs", input);
3878
3879 assert!(
3880 document
3881 .accesses
3882 .iter()
3883 .any(|access| { access.table == "users" && access.role == DataAccessRole::Reader })
3884 );
3885 assert!(
3886 document
3887 .accesses
3888 .iter()
3889 .any(|access| { access.table == "teams" && access.role == DataAccessRole::Reader })
3890 );
3891 assert!(
3892 document.incomplete
3893 && document.warnings.contains(&DataWarning::DynamicQuery)
3894 && document.frameworks == vec![DataFramework::Sqlx]
3895 );
3896 }
3897
3898 #[test]
3899 fn standalone_query_files_are_not_misclassified_as_migrations() {
3900 let document = extract_data_artifact(
3901 "queries/find_users.sql",
3902 "SELECT id FROM users WHERE active = true;",
3903 )
3904 .expect("query file should parse");
3905
3906 assert!(
3907 document.artifact_kind == DataArtifactKind::SqlQueryFile
3908 && document.migration.is_none()
3909 && document.accesses.iter().any(|access| {
3910 access.table == "users" && access.role == DataAccessRole::Reader
3911 })
3912 );
3913 }
3914
3915 #[test]
3916 fn sqlx_reversible_migration_filename_sets_version_metadata() {
3917 let document = extract_data_artifact(
3918 "migrations/0042_add_users.up.sql",
3919 "CREATE TABLE users (id INTEGER);",
3920 )
3921 .expect("migration should parse");
3922 let migration = document.migration.expect("migration metadata");
3923
3924 assert!(
3925 migration.order_hint == Some(42)
3926 && migration.revision.as_deref() == Some("42")
3927 && migration.reversible
3928 );
3929 }
3930
3931 #[test]
3932 fn literal_write_records_writer_role() {
3933 let input = r#"
3934async def save(conn):
3935 await conn.execute("UPDATE users SET active = ? WHERE id = ?", True, 7)
3936"#;
3937
3938 let document = parse_literal_sql_source(SourceLanguage::Python, "users.py", input);
3939
3940 assert!(document.accesses.iter().any(|access| {
3941 access.role == DataAccessRole::Writer
3942 && access.operation == Some(DataOperation::Update)
3943 && access.table == "users"
3944 }));
3945 }
3946
3947 #[test]
3948 fn pymysql_wrapper_f_string_preserves_static_table_access() {
3949 let input = r#"
3950from src.clases.Database import Database
3951
3952def load(account_id):
3953 return Database.sql(f"SELECT ID FROM accounts WHERE ID = {account_id}")
3954"#;
3955
3956 let document = parse_literal_sql_source(SourceLanguage::Python, "accounts.py", input);
3957
3958 assert!(document.accesses.iter().any(|access| {
3959 access.role == DataAccessRole::Reader
3960 && access.operation == Some(DataOperation::Select)
3961 && access.table == "accounts"
3962 }));
3963 assert!(
3964 document.incomplete && document.warnings.contains(&DataWarning::DynamicQuery),
3965 "interpolated values must remain explicitly incomplete"
3966 );
3967 }
3968
3969 #[test]
3970 fn pymysql_wrapper_does_not_promote_interpolated_table_names() {
3971 let input = r#"
3972from src.clases.Database import Database
3973
3974def load(table):
3975 return Database.sql(f"SELECT ID FROM {table} WHERE active = 1")
3976"#;
3977
3978 let document = parse_literal_sql_source(SourceLanguage::Python, "accounts.py", input);
3979
3980 assert!(
3981 document.accesses.is_empty()
3982 && document.incomplete
3983 && document.warnings.contains(&DataWarning::DynamicQuery)
3984 );
3985 }
3986
3987 #[test]
3988 fn psycopg_format_values_preserve_static_table_access() {
3989 let input = r#"
3990import psycopg2
3991
3992def search(cursor, value):
3993 cursor.execute("""
3994 SELECT id FROM embeddings WHERE content = '{}'
3995 """.format(value))
3996"#;
3997
3998 let document = parse_literal_sql_source(SourceLanguage::Python, "vectors.py", input);
3999
4000 assert!(document.frameworks.contains(&DataFramework::Psycopg));
4001 assert!(document.accesses.iter().any(|access| {
4002 access.role == DataAccessRole::Reader
4003 && access.table == "embeddings"
4004 && access.owner.as_deref() == Some("search")
4005 }));
4006 assert!(document.warnings.contains(&DataWarning::DynamicQuery));
4007 }
4008
4009 #[test]
4010 fn sqlalchemy_source_records_model_reads_and_writes_without_guessing_tables() {
4011 let input = r#"
4012from sqlalchemy.orm import Session
4013from app import models
4014
4015def create(db: Session):
4016 service = models.Services(name="test")
4017 db.add(service)
4018
4019def list_all(db: Session):
4020 return db.query(models.Services).all()
4021"#;
4022
4023 let document = parse_literal_sql_source(SourceLanguage::Python, "controllers.py", input);
4024
4025 assert!(document.frameworks.contains(&DataFramework::SqlAlchemy));
4026 assert!(document.accesses.iter().any(|access| {
4027 access.role == DataAccessRole::Writer
4028 && access.model.as_deref() == Some("Services")
4029 && access.operation == Some(DataOperation::Insert)
4030 }));
4031 assert!(document.accesses.iter().any(|access| {
4032 access.role == DataAccessRole::Reader
4033 && access.model.as_deref() == Some("Services")
4034 && access.operation == Some(DataOperation::Select)
4035 }));
4036 assert!(
4037 document
4038 .accesses
4039 .iter()
4040 .all(|access| access.table.is_empty())
4041 );
4042 }
4043
4044 #[test]
4045 fn mariadb_dump_recovers_create_tables_after_unsupported_statements() {
4046 let input = r"
4047/*!40101 SET NAMES utf8mb4 */;
4048CREATE DATABASE IF NOT EXISTS `processor`;
4049USE `processor`;
4050
4051CREATE TABLE IF NOT EXISTS `accounts` (
4052 `ID` int(10) unsigned NOT NULL AUTO_INCREMENT,
4053 `number` int(10) unsigned NOT NULL DEFAULT 0,
4054 PRIMARY KEY (`ID`),
4055 UNIQUE KEY `number_unique` (`number`)
4056) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
4057
4058LOCK TABLES `accounts` WRITE;
4059";
4060
4061 let document =
4062 extract_data_artifact("sql/base_embozador.sql", input).expect("dump should recover");
4063
4064 assert!(document.tables.iter().any(|table| {
4065 table.name == "accounts"
4066 && table.columns.iter().any(|column| column.name == "ID")
4067 && table.columns.iter().any(|column| column.name == "number")
4068 }));
4069 assert!(
4070 document.incomplete
4071 && document.warnings.iter().any(|warning| {
4072 matches!(
4073 warning,
4074 DataWarning::SqlParseRecovery | DataWarning::UnsupportedConstruct
4075 )
4076 }),
4077 "recovered dumps must retain the fact that unsupported statements were skipped"
4078 );
4079 }
4080
4081 #[test]
4082 fn literal_queries_cover_remaining_source_languages() {
4083 let cases = [
4084 (
4085 SourceLanguage::JavaScript,
4086 r#"function load() { db.query("SELECT id FROM users"); }"#,
4087 ),
4088 (
4089 SourceLanguage::TypeScript,
4090 r#"const load = () => db.query("SELECT id FROM users");"#,
4091 ),
4092 (
4093 SourceLanguage::Go,
4094 "func load() { db.Query(`SELECT id FROM users`) }",
4095 ),
4096 (
4097 SourceLanguage::Java,
4098 r#"void load() { statement.executeQuery("SELECT id FROM users"); }"#,
4099 ),
4100 ];
4101
4102 assert!(cases.into_iter().all(|(language, source)| {
4103 parse_literal_sql_source(language, "source.file", source)
4104 .accesses
4105 .iter()
4106 .any(|access| access.table == "users" && access.role == DataAccessRole::Reader)
4107 }));
4108 }
4109
4110 #[test]
4111 fn literal_query_context_should_preserve_utf8_boundaries() {
4112 let input = format!(
4113 "{} db.query(\"SELECT id FROM users\") {}",
4114 "─".repeat(70),
4115 "─".repeat(40)
4116 );
4117
4118 let document = parse_literal_sql_source(SourceLanguage::JavaScript, "src/users.js", &input);
4119
4120 assert!(
4121 document
4122 .accesses
4123 .iter()
4124 .any(|access| access.table == "users")
4125 );
4126 }
4127
4128 #[test]
4129 fn dynamic_query_is_incomplete_and_unlinked() {
4130 let input = r#"
4131fn load(pool: &Pool, table: &str) {
4132 let sql = "SELECT * FROM ".to_owned() + table;
4133 sqlx::query(&sql);
4134}
4135"#;
4136
4137 let document = parse_literal_sql_source(SourceLanguage::Rust, "src/dynamic.rs", input);
4138
4139 assert!(
4140 document.incomplete
4141 && document.accesses.is_empty()
4142 && document.warnings.contains(&DataWarning::DynamicQuery)
4143 );
4144 }
4145
4146 #[test]
4147 fn serialization_does_not_persist_secrets_or_sql_bodies() {
4148 let input = r"
4149-- postgres://admin:credential@localhost/private
4150CREATE TABLE users (
4151 id INTEGER,
4152 token TEXT DEFAULT 'top-secret-token'
4153);
4154";
4155 let document =
4156 extract_data_artifact("schema.sql", input).expect("schema should parse safely");
4157 let serialized = serde_json::to_string(&document).expect("document should serialize");
4158
4159 assert!(
4160 !serialized.contains("credential")
4161 && !serialized.contains("top-secret-token")
4162 && !serialized.contains("CREATE TABLE")
4163 && !serialized.contains("postgres://")
4164 );
4165 }
4166
4167 #[test]
4168 fn literal_query_serialization_discards_values_and_body() {
4169 let input = r#"db.query("SELECT id FROM users WHERE token = 'literal-secret-value'")"#;
4170 let document = parse_literal_sql_source(SourceLanguage::JavaScript, "src/users.js", input);
4171 let serialized = serde_json::to_string(&document).expect("document should serialize");
4172
4173 assert!(
4174 !serialized.contains("literal-secret-value")
4175 && !serialized.contains("SELECT id")
4176 && serialized.contains("users")
4177 );
4178 }
4179
4180 #[test]
4181 fn parser_recovery_is_incomplete_without_echoing_secret_input() {
4182 let document = extract_data_artifact(
4183 "schema.sql",
4184 "CREATE TABLE secret (token DEFAULT 'do-not-echo'",
4185 )
4186 .expect("unsupported SQL syntax should degrade");
4187 let serialized = serde_json::to_string(&document).expect("document should serialize");
4188
4189 assert!(
4190 document.incomplete
4191 && document.warnings.contains(&DataWarning::SqlParseRecovery)
4192 && !serialized.contains("do-not-echo")
4193 );
4194 }
4195
4196 #[test]
4197 fn output_is_deterministic_and_deduplicated() {
4198 let input = "CREATE TABLE b (id INT); CREATE TABLE a (id INT);";
4199 let first = extract_data_artifact("schema.sql", input).expect("schema should parse");
4200 let second = extract_data_artifact("schema.sql", input).expect("schema should parse");
4201
4202 assert_eq!(first, second);
4203 }
4204
4205 #[test]
4206 fn unsupported_filename_is_rejected_without_path_echo() {
4207 let error = extract_data_artifact("secrets/config.txt", "password=private")
4208 .expect_err("unsupported file should fail");
4209
4210 assert_eq!(error, DataExtractionError::UnsupportedArtifact);
4211 }
4212
4213 #[test]
4214 fn oversized_input_returns_bounded_incomplete_document() {
4215 let input = "x".repeat(MAX_DATA_INPUT_BYTES + 1);
4216 let document =
4217 extract_data_artifact("schema.sql", &input).expect("oversized SQL should degrade");
4218
4219 assert!(
4220 document.incomplete
4221 && document.warnings == vec![DataWarning::LimitExceeded]
4222 && document.tables.is_empty()
4223 );
4224 }
4225
4226 #[test]
4227 fn oversized_schema_retains_tables_and_truncates_details() {
4228 let mut document = empty_document("schema.sql", DataArtifactKind::DeclarativeSqlSchema);
4229 for table_index in 0..5 {
4230 let mut table = table_from_qualified_text(
4231 &format!("table_{table_index}"),
4232 evidence(table_index + 1),
4233 );
4234 table.columns = (0..1_000)
4235 .map(|column_index| DatabaseColumn {
4236 name: format!("column_{column_index}"),
4237 data_type: Some("integer".to_owned()),
4238 nullable: None,
4239 primary_key: false,
4240 unique: false,
4241 default_present: false,
4242 evidence: evidence(table_index + 1),
4243 })
4244 .collect();
4245 document.tables.push(table);
4246 }
4247
4248 finish_document(&mut document);
4249
4250 assert!(
4251 document.tables.len() == 5
4252 && document
4253 .tables
4254 .iter()
4255 .all(|table| table.name.starts_with("table_"))
4256 && structured_item_count(&document) == MAX_DATA_ITEMS
4257 && document.incomplete
4258 && document.warnings.contains(&DataWarning::LimitExceeded)
4259 );
4260 }
4261
4262 #[test]
4263 fn evidence_line_enforces_bounds() {
4264 assert!(DataEvidenceLine::new(0).is_err());
4265 assert!(DataEvidenceLine::new(MAX_DATA_SOURCE_LINES).is_ok());
4266 assert!(DataEvidenceLine::new(MAX_DATA_SOURCE_LINES + 1).is_err());
4267 }
4268}