data_source/
lib.rs

1pub mod embedded {
2    include!(concat!(env!("OUT_DIR"), "/embedded_data.rs"));
3}
4
5#[cfg(feature = "uniffi-bindings")]
6uniffi::setup_scaffolding!();
7
8pub mod additional_schema_source;
9#[cfg(feature = "maintenance")]
10pub mod additional_specs;
11#[cfg(feature = "maintenance")]
12pub mod cfnlint_tables;
13#[cfg(feature = "maintenance")]
14pub mod codegen_cel;
15#[cfg(feature = "maintenance")]
16pub mod codegen_schema_validator;
17pub mod compiled_schema;
18#[cfg(feature = "maintenance")]
19pub mod extensions;
20#[cfg(feature = "maintenance")]
21pub mod process;
22#[cfg(feature = "maintenance")]
23pub mod regions;
24pub mod rule_data;
25#[cfg(feature = "maintenance")]
26pub mod schema;
27#[cfg(feature = "maintenance")]
28mod source_versions;
29pub mod types;
30
31pub use additional_schema_source::{AdditionalSchemaSource, SchemaSourceError};
32
33#[cfg(feature = "maintenance")]
34use log::{error, info};
35#[cfg(feature = "maintenance")]
36use std::fs;
37#[cfg(feature = "maintenance")]
38use std::path::{Path, PathBuf};
39
40/// Convert a rule-source resource type directory name (e.g., "aws_rds_dbinstance")
41/// to the normalized form used by this project (e.g., "aws-rds-dbinstance").
42#[cfg(feature = "maintenance")]
43pub fn rule_source_dir_to_name(dir_name: &str) -> String {
44    dir_name.to_lowercase().replace('_', "-")
45}
46
47/// Resolve the rule-source root directory, returning an error if not provided or missing.
48#[cfg(feature = "maintenance")]
49pub fn resolve_rule_source_dir(path: Option<&str>) -> anyhow::Result<PathBuf> {
50    let p = path.ok_or_else(|| anyhow::anyhow!("Pass --cfn-lint-root to the cfn-lint repo root"))?;
51    let pb = PathBuf::from(p);
52    anyhow::ensure!(pb.exists(), "Rule-source directory not found: {}", pb.display());
53    Ok(pb)
54}
55
56#[cfg(feature = "maintenance")]
57#[derive(Debug, Default)]
58pub struct SyncStats {
59    pub files_written: usize,
60    pub files_skipped: usize,
61    pub errors: Vec<String>,
62}
63
64#[cfg(feature = "maintenance")]
65impl SyncStats {
66    pub fn log(&self, label: &str) {
67        info!(
68            "{}: {} written, {} skipped, {} errors",
69            label,
70            self.files_written,
71            self.files_skipped,
72            self.errors.len()
73        );
74        for e in &self.errors {
75            error!("{}: {}", label, e);
76        }
77    }
78
79    pub fn fail_on_errors(&self, label: &str) -> anyhow::Result<()> {
80        self.log(label);
81        if !self.errors.is_empty() {
82            anyhow::bail!("{} completed with {} error(s):\n  {}", label, self.errors.len(), self.errors.join("\n  "));
83        }
84        Ok(())
85    }
86}
87
88#[cfg(feature = "maintenance")]
89fn write_source_versions(path: &Path, versions: source_versions::SourceVersions) -> anyhow::Result<()> {
90    let versions = source_versions::SourceVersions::new(versions.cfn_lint_version, versions.resource_schema_version)
91        .map_err(anyhow::Error::msg)?;
92    let mut contents = serde_json::to_string_pretty(&versions)?;
93    contents.push('\n');
94    fs::write(path, contents)?;
95    Ok(())
96}
97
98#[cfg(feature = "maintenance")]
99pub fn sync_upstream(upstream_dir: &Path, rule_source_root: &str) -> anyhow::Result<()> {
100    info!("=== Sync phase ===");
101
102    let generated_root = upstream_dir
103        .parent()
104        .ok_or_else(|| anyhow::anyhow!("upstream directory must have a parent: {}", upstream_dir.display()))?;
105    let generated_data = generated_root.join("generated").join("data");
106    fs::create_dir_all(&generated_data)?;
107    let source_versions_path = generated_data.join(source_versions::SOURCE_VERSIONS_FILE);
108    let rule_source_dir = resolve_rule_source_dir(Some(rule_source_root))?;
109
110    info!("Step 1: Downloading enhanced CloudFormation schemas (fully patched, with region maps)");
111    let (schema_stats, resource_schema_version) = schema::download_schemas(upstream_dir)?;
112    schema_stats.fail_on_errors("Download")?;
113
114    info!("Step 2: Building region resource types from downloaded provider maps");
115    regions::sync_regions(&schema::providers_dir(upstream_dir), &generated_data)?.fail_on_errors("Regions")?;
116
117    info!("Step 3: Syncing extensions from {}", rule_source_dir.display());
118    extensions::sync_extensions(&rule_source_dir, &upstream_dir.join("extensions"), &generated_data)?
119        .fail_on_errors("Extensions")?;
120
121    info!("Step 4: Syncing additional specs from {}", rule_source_dir.display());
122    additional_specs::sync_additional_specs(&rule_source_dir, &generated_data, upstream_dir)?
123        .fail_on_errors("AdditionalSpecs")?;
124
125    info!("Step 5: Extracting data tables embedded in cfn-lint rule code");
126    let (table_stats, cfn_lint_version) = cfnlint_tables::sync_cfnlint_tables(&rule_source_dir, &generated_data)?;
127    table_stats.fail_on_errors("CfnLintTables")?;
128
129    verify_files_exist_and_populated(REQUIRED_SYNC_FILES, &generated_data, "Sync")?;
130    let source_versions =
131        source_versions::SourceVersions::new(cfn_lint_version, resource_schema_version).map_err(anyhow::Error::msg)?;
132    write_source_versions(&source_versions_path, source_versions)?;
133    info!("Recorded complete data source provenance in {}", source_versions_path.display());
134
135    Ok(())
136}
137
138#[cfg(feature = "maintenance")]
139pub fn generate_all(upstream_dir: &Path, generated_dir: &Path, handwritten_dir: &Path) -> anyhow::Result<()> {
140    info!("=== Generate phase ===");
141
142    info!("Step 1: Processing schemas (extensions, metadata)");
143    process::process_schemas(upstream_dir, generated_dir, handwritten_dir)?.fail_on_errors("Process")?;
144
145    info!("Step 2: Generating CEL rules");
146    codegen_cel::generate(generated_dir, handwritten_dir)?;
147
148    info!("Step 3: Generating compiled schemas for schema-validator");
149    codegen_schema_validator::generate(generated_dir, upstream_dir)?;
150
151    info!("Step 4: Verifying all expected output files");
152    verify_outputs(generated_dir, handwritten_dir)?;
153
154    Ok(())
155}
156
157/// Expected data files that must exist and contain real data after a
158/// successful sync+generate.
159
160/// Files produced by sync_upstream (extensions, regions, additional specs).
161#[cfg(feature = "maintenance")]
162const REQUIRED_SYNC_FILES: &[&str] = &[
163    // Enum data documents (from the upstream rule-source extensions)
164    "aws_amazonmq_broker_instancetype_enum",
165    "aws_appstream_fleet_instancetype_enum",
166    "aws_dax_cluster_nodetype_enum",
167    "aws_docdb_dbinstance_dbinstanceclass_enum",
168    "aws_ec2_instance_instancetype_enum",
169    "aws_elasticache_cachecluster_cachenodetype_enum",
170    "aws_emr_cluster_instancetypeconfig_instancetype_enum",
171    "aws_gamelift_fleet_ec2instancetype_enum",
172    "aws_managedblockchain_node_nodeconfiguration_instancetype_enum",
173    "aws_neptune_dbinstance_dbinstanceclass_enum",
174    "aws_rds_dbcluster_dbclusterinstanceclass_enum",
175    "aws_rds_dbinstance_db_instance_class",
176    "aws_rds_dbinstance_dbinstanceclass_enum",
177    "aws_redshift_cluster_nodetype_enum",
178    "aws_sagemaker_processing_instancetype_enum",
179    "aws_sagemaker_hosting_instancetype_enum",
180    "aws_sagemaker_transform_instancetype_enum",
181    "aws_sagemaker_cluster_instancetype_enum",
182    "aws_elasticsearch_domain_elasticsearchclusterconfig_instancetype_enum",
183    "aws_opensearchservice_domain_clusterconfig_instancetype_enum",
184    // Additional specs
185    "iam_action_resource_patterns",
186    "lambda_runtimes",
187    "region_resource_types",
188    "stateful_resource_types",
189    // Tables extracted from cfn-lint rule code
190    "getatt_additions",
191    "retention_period_requirements",
192    "codepipeline_action_artifact_counts",
193    "cfnlint_rule_tables",
194];
195
196/// Data files produced by generate_all schema processing.
197#[cfg(feature = "maintenance")]
198const REQUIRED_GENERATE_DATA_FILES: &[&str] =
199    &["resource_lifecycle", "schema_metadata", "primary_identifiers", "getatt_attributes", "known_resource_types"];
200
201/// Schema-validator files produced by generate_all.
202#[cfg(feature = "maintenance")]
203const REQUIRED_SCHEMA_VALIDATOR_FILES: &[&str] = &["compiled_schemas", "ref_types", "extensions", "region_enums"];
204
205#[cfg(feature = "maintenance")]
206const REQUIRED_CEL_RULE_FILES: &[&str] = &["generated_rules"];
207
208#[cfg(feature = "maintenance")]
209const REQUIRED_HANDWRITTEN_FILES: &[&str] = &[
210    "deprecated_resource_types",
211    "getatt_return_type_overrides",
212    "schema_dependent_excluded_overrides",
213    "secretsmanager_arn_fields",
214    "sensitive_ports",
215];
216
217#[cfg(feature = "maintenance")]
218fn verify_sync_outputs(data_dir: &Path) -> anyhow::Result<()> {
219    source_versions::SourceVersions::read(&data_dir.join(source_versions::SOURCE_VERSIONS_FILE))
220        .map_err(anyhow::Error::msg)?;
221    verify_files_exist_and_populated(REQUIRED_SYNC_FILES, data_dir, "Sync")
222}
223
224#[cfg(feature = "maintenance")]
225fn verify_outputs(generated_dir: &Path, handwritten_dir: &Path) -> anyhow::Result<()> {
226    let data_dir = generated_dir.join("data");
227    let schema_validator_dir = generated_dir.join("schema-validator");
228    let cel_rules_dir = generated_dir.join("cel-rules");
229
230    verify_sync_outputs(&data_dir)?;
231    verify_files_exist_and_populated(REQUIRED_GENERATE_DATA_FILES, &data_dir, "Generate data")?;
232    verify_files_exist_and_populated(REQUIRED_SCHEMA_VALIDATOR_FILES, &schema_validator_dir, "Schema validator")?;
233    verify_files_exist_and_populated(REQUIRED_CEL_RULE_FILES, &cel_rules_dir, "Generated CEL rules")?;
234    verify_files_exist_and_populated(REQUIRED_HANDWRITTEN_FILES, handwritten_dir, "Handwritten")?;
235
236    let total = REQUIRED_SYNC_FILES.len()
237        + REQUIRED_GENERATE_DATA_FILES.len()
238        + REQUIRED_SCHEMA_VALIDATOR_FILES.len()
239        + REQUIRED_CEL_RULE_FILES.len()
240        + REQUIRED_HANDWRITTEN_FILES.len()
241        + 1;
242    info!("Verified {total} required data files");
243    Ok(())
244}
245
246#[cfg(feature = "maintenance")]
247fn verify_files_exist_and_populated(names: &[&str], directory: &Path, label: &str) -> anyhow::Result<()> {
248    let mut missing = Vec::new();
249    let mut stubs = Vec::new();
250
251    for name in names {
252        let path = directory.join(format!("{name}.json"));
253        if !path.is_file() {
254            missing.push(name.to_string());
255            continue;
256        }
257
258        if is_stub(&path) {
259            stubs.push(name.to_string());
260        }
261    }
262
263    if !missing.is_empty() || !stubs.is_empty() {
264        let mut msg = String::new();
265        if !missing.is_empty() {
266            msg.push_str(&format!("  Missing: {}\n", missing.join(", ")));
267        }
268        if !stubs.is_empty() {
269            msg.push_str(&format!("  Empty stubs: {}\n", stubs.join(", ")));
270        }
271        anyhow::bail!(
272            "{label} verification failed - {}/{} files not populated:\n{msg}",
273            missing.len() + stubs.len(),
274            names.len()
275        );
276    }
277    Ok(())
278}
279
280/// A file is an invalid empty stub if it contains only `{}` or `[]`.
281#[cfg(feature = "maintenance")]
282fn is_stub(path: &Path) -> bool {
283    match fs::read_to_string(path) {
284        Ok(content) => {
285            let trimmed = content.trim();
286            trimmed == "{}" || trimmed == "[]"
287        }
288        Err(_) => true,
289    }
290}
data_source/lib.rs

data_source/
lib.rs
