data_source/
lib.rs

1pub mod embedded {
2    include!(concat!(env!("OUT_DIR"), "/embedded_data.rs"));
3}
4
5#[cfg(feature = "uniffi-bindings")]
6uniffi::setup_scaffolding!();
7
8pub mod additional_schema_source;
9#[cfg(feature = "maintenance")]
10pub mod additional_specs;
11#[cfg(feature = "maintenance")]
12pub mod aws_cli_catalog;
13#[cfg(feature = "maintenance")]
14pub mod cfnlint_tables;
15#[cfg(feature = "maintenance")]
16pub mod codegen_cel;
17#[cfg(feature = "maintenance")]
18pub mod codegen_schema_validator;
19pub mod compiled_schema;
20#[cfg(feature = "maintenance")]
21pub mod extensions;
22#[cfg(feature = "maintenance")]
23pub mod process;
24#[cfg(feature = "maintenance")]
25pub mod regions;
26pub mod rule_data;
27#[cfg(feature = "maintenance")]
28pub mod schema;
29#[cfg(feature = "maintenance")]
30mod source_versions;
31pub mod types;
32
33pub use additional_schema_source::{AdditionalSchemaSource, SchemaSourceError};
34#[cfg(feature = "maintenance")]
35pub use aws_cli_catalog::{
36    AWS_CLI_OPERATION_CATALOG_FILE, PreservedAwsCliCatalog, generate_aws_cli_catalog, preserve_aws_cli_catalog,
37    restore_aws_cli_catalog,
38};
39
40#[cfg(feature = "maintenance")]
41use log::{error, info};
42#[cfg(feature = "maintenance")]
43use std::fs;
44#[cfg(feature = "maintenance")]
45use std::path::{Path, PathBuf};
46
47/// Convert a rule-source resource type directory name (e.g., "aws_rds_dbinstance")
48/// to the normalized form used by this project (e.g., "aws-rds-dbinstance").
49#[cfg(feature = "maintenance")]
50pub fn rule_source_dir_to_name(dir_name: &str) -> String {
51    dir_name.to_lowercase().replace('_', "-")
52}
53
54/// Resolve the rule-source root directory, returning an error if not provided or missing.
55#[cfg(feature = "maintenance")]
56pub fn resolve_rule_source_dir(path: Option<&str>) -> anyhow::Result<PathBuf> {
57    let p = path.ok_or_else(|| anyhow::anyhow!("Pass --cfn-lint-root to the cfn-lint repo root"))?;
58    let pb = PathBuf::from(p);
59    anyhow::ensure!(pb.exists(), "Rule-source directory not found: {}", pb.display());
60    Ok(pb)
61}
62
63#[cfg(feature = "maintenance")]
64#[derive(Debug, Default)]
65pub struct SyncStats {
66    pub files_written: usize,
67    pub files_skipped: usize,
68    pub errors: Vec<String>,
69}
70
71#[cfg(feature = "maintenance")]
72impl SyncStats {
73    pub fn log(&self, label: &str) {
74        info!(
75            "{}: {} written, {} skipped, {} errors",
76            label,
77            self.files_written,
78            self.files_skipped,
79            self.errors.len()
80        );
81        for e in &self.errors {
82            error!("{}: {}", label, e);
83        }
84    }
85
86    pub fn fail_on_errors(&self, label: &str) -> anyhow::Result<()> {
87        self.log(label);
88        if !self.errors.is_empty() {
89            anyhow::bail!("{} completed with {} error(s):\n  {}", label, self.errors.len(), self.errors.join("\n  "));
90        }
91        Ok(())
92    }
93}
94
95/// Manifest writers. They live here rather than in `source_versions.rs` because
96/// the build script compiles that file too and only ever reads the manifest.
97#[cfg(feature = "maintenance")]
98impl source_versions::SourceVersions {
99    pub(crate) fn new(
100        cfn_lint_version: String,
101        resource_schema_version: String,
102        aws_cli_version: Option<String>,
103    ) -> Result<Self, String> {
104        let versions = Self { cfn_lint_version, resource_schema_version, aws_cli_version };
105        versions.validate()?;
106        Ok(versions)
107    }
108
109    /// The manifest with the AWS CLI entry recorded and the `sync`-owned entries kept.
110    pub(crate) fn with_aws_cli_version(self, aws_cli_version: String) -> Result<Self, String> {
111        Self::new(self.cfn_lint_version, self.resource_schema_version, Some(aws_cli_version))
112    }
113}
114
115#[cfg(feature = "maintenance")]
116pub(crate) fn write_source_versions(path: &Path, versions: source_versions::SourceVersions) -> anyhow::Result<()> {
117    let mut contents = serde_json::to_string_pretty(&versions)?;
118    contents.push('\n');
119    fs::write(path, contents)?;
120    Ok(())
121}
122
123#[cfg(feature = "maintenance")]
124pub fn sync_upstream(upstream_dir: &Path, rule_source_root: &str) -> anyhow::Result<()> {
125    info!("=== Sync phase ===");
126
127    let generated_root = upstream_dir
128        .parent()
129        .ok_or_else(|| anyhow::anyhow!("upstream directory must have a parent: {}", upstream_dir.display()))?;
130    let generated_data = generated_root.join("generated").join("data");
131    fs::create_dir_all(&generated_data)?;
132    let source_versions_path = generated_data.join(source_versions::SOURCE_VERSIONS_FILE);
133    let rule_source_dir = resolve_rule_source_dir(Some(rule_source_root))?;
134
135    info!("Step 1: Downloading enhanced CloudFormation schemas (fully patched, with region maps)");
136    let (schema_stats, resource_schema_version) = schema::download_schemas(upstream_dir)?;
137    schema_stats.fail_on_errors("Download")?;
138
139    info!("Step 2: Building region resource types from downloaded provider maps");
140    regions::sync_regions(&schema::providers_dir(upstream_dir), &generated_data)?.fail_on_errors("Regions")?;
141
142    info!("Step 3: Syncing extensions from {}", rule_source_dir.display());
143    extensions::sync_extensions(&rule_source_dir, &upstream_dir.join("extensions"), &generated_data)?
144        .fail_on_errors("Extensions")?;
145
146    info!("Step 4: Syncing additional specs from {}", rule_source_dir.display());
147    additional_specs::sync_additional_specs(&rule_source_dir, &generated_data, upstream_dir)?
148        .fail_on_errors("AdditionalSpecs")?;
149
150    info!("Step 5: Extracting data tables embedded in cfn-lint rule code");
151    let (table_stats, cfn_lint_version) =
152        cfnlint_tables::sync_cfnlint_tables(&rule_source_dir, &generated_data, upstream_dir)?;
153    table_stats.fail_on_errors("CfnLintTables")?;
154
155    verify_files_exist_and_populated(REQUIRED_SYNC_FILES, &generated_data, "Sync")?;
156    verify_files_exist_and_populated(REQUIRED_UPSTREAM_FILES, upstream_dir, "Sync")?;
157    // `generated/data` was cleared ahead of this sync, so the manifest is written
158    // fresh; the AWS CLI entry is recorded by the final `sync` step, which either
159    // regenerates the catalog or restores the preserved one.
160    let source_versions = source_versions::SourceVersions::new(cfn_lint_version, resource_schema_version, None)
161        .map_err(anyhow::Error::msg)?;
162    write_source_versions(&source_versions_path, source_versions)?;
163    info!("Recorded complete data source provenance in {}", source_versions_path.display());
164
165    Ok(())
166}
167
168#[cfg(feature = "maintenance")]
169pub fn generate_all(upstream_dir: &Path, generated_dir: &Path, handwritten_dir: &Path) -> anyhow::Result<()> {
170    info!("=== Generate phase ===");
171
172    info!("Step 1: Processing schemas (extensions, metadata)");
173    process::process_schemas(upstream_dir, generated_dir, handwritten_dir)?.fail_on_errors("Process")?;
174
175    info!("Step 2: Generating CEL rules");
176    codegen_cel::generate(generated_dir, handwritten_dir)?;
177
178    info!("Step 3: Generating compiled schemas for schema-validator");
179    codegen_schema_validator::generate(generated_dir, upstream_dir)?;
180
181    info!("Step 4: Verifying all expected output files");
182    verify_outputs(upstream_dir, generated_dir, handwritten_dir)?;
183
184    Ok(())
185}
186
187/// Expected data files that must exist and contain real data after a
188/// successful sync+generate.
189
190/// Files produced by sync_upstream (extensions, regions, additional specs).
191#[cfg(feature = "maintenance")]
192const REQUIRED_SYNC_FILES: &[&str] = &[
193    // Enum data documents (from the upstream rule-source extensions)
194    "aws_amazonmq_broker_instancetype_enum",
195    "aws_appstream_fleet_instancetype_enum",
196    "aws_dax_cluster_nodetype_enum",
197    "aws_docdb_dbinstance_dbinstanceclass_enum",
198    "aws_ec2_instance_instancetype_enum",
199    "aws_elasticache_cachecluster_cachenodetype_enum",
200    "aws_emr_cluster_instancetypeconfig_instancetype_enum",
201    "aws_gamelift_fleet_ec2instancetype_enum",
202    "aws_managedblockchain_node_nodeconfiguration_instancetype_enum",
203    "aws_neptune_dbinstance_dbinstanceclass_enum",
204    "aws_rds_dbcluster_dbclusterinstanceclass_enum",
205    "aws_rds_dbinstance_db_instance_class",
206    "aws_rds_dbinstance_dbinstanceclass_enum",
207    "aws_redshift_cluster_nodetype_enum",
208    "aws_sagemaker_processing_instancetype_enum",
209    "aws_sagemaker_hosting_instancetype_enum",
210    "aws_sagemaker_transform_instancetype_enum",
211    "aws_sagemaker_cluster_instancetype_enum",
212    "aws_elasticsearch_domain_elasticsearchclusterconfig_instancetype_enum",
213    "aws_opensearchservice_domain_clusterconfig_instancetype_enum",
214    // Additional specs
215    "iam_action_resource_patterns",
216    "lambda_runtimes",
217    "region_resource_types",
218    "stateful_resource_types",
219    // Tables extracted from cfn-lint rule code
220    "retention_period_requirements",
221    "codepipeline_action_artifact_counts",
222    "cfnlint_rule_tables",
223];
224
225/// Raw intermediates produced by sync_upstream into the upstream directory. They
226/// are consumed only by generate_all and are never embedded, so they are not
227/// committed with the generated data.
228#[cfg(feature = "maintenance")]
229const REQUIRED_UPSTREAM_FILES: &[&str] = &[cfnlint_tables::GETATT_ADDITIONS_NAME];
230
231/// Data files produced by generate_all schema processing.
232#[cfg(feature = "maintenance")]
233const REQUIRED_GENERATE_DATA_FILES: &[&str] =
234    &["resource_lifecycle", "schema_metadata", "primary_identifiers", "getatt_attributes", "known_resource_types"];
235
236/// Schema-validator files produced by generate_all.
237#[cfg(feature = "maintenance")]
238const REQUIRED_SCHEMA_VALIDATOR_FILES: &[&str] = &["compiled_schemas", "ref_types", "extensions", "region_enums"];
239
240#[cfg(feature = "maintenance")]
241const REQUIRED_CEL_RULE_FILES: &[&str] = &["generated_rules"];
242
243#[cfg(feature = "maintenance")]
244const REQUIRED_HANDWRITTEN_FILES: &[&str] = &[
245    "deprecated_resource_types",
246    "getatt_return_type_overrides",
247    "schema_dependent_excluded_overrides",
248    "secretsmanager_arn_fields",
249    "sensitive_ports",
250];
251
252#[cfg(feature = "maintenance")]
253fn verify_sync_outputs(upstream_dir: &Path, data_dir: &Path) -> anyhow::Result<()> {
254    source_versions::SourceVersions::read(&data_dir.join(source_versions::SOURCE_VERSIONS_FILE))
255        .map_err(anyhow::Error::msg)?;
256    verify_files_exist_and_populated(REQUIRED_SYNC_FILES, data_dir, "Sync")?;
257    verify_files_exist_and_populated(REQUIRED_UPSTREAM_FILES, upstream_dir, "Sync")
258}
259
260#[cfg(feature = "maintenance")]
261fn verify_outputs(upstream_dir: &Path, generated_dir: &Path, handwritten_dir: &Path) -> anyhow::Result<()> {
262    let data_dir = generated_dir.join("data");
263    let schema_validator_dir = generated_dir.join("schema-validator");
264    let cel_rules_dir = generated_dir.join("cel-rules");
265
266    verify_sync_outputs(upstream_dir, &data_dir)?;
267    verify_files_exist_and_populated(REQUIRED_GENERATE_DATA_FILES, &data_dir, "Generate data")?;
268    verify_files_exist_and_populated(REQUIRED_SCHEMA_VALIDATOR_FILES, &schema_validator_dir, "Schema validator")?;
269    verify_files_exist_and_populated(REQUIRED_CEL_RULE_FILES, &cel_rules_dir, "Generated CEL rules")?;
270    verify_files_exist_and_populated(REQUIRED_HANDWRITTEN_FILES, handwritten_dir, "Handwritten")?;
271
272    let total = REQUIRED_SYNC_FILES.len()
273        + REQUIRED_UPSTREAM_FILES.len()
274        + REQUIRED_GENERATE_DATA_FILES.len()
275        + REQUIRED_SCHEMA_VALIDATOR_FILES.len()
276        + REQUIRED_CEL_RULE_FILES.len()
277        + REQUIRED_HANDWRITTEN_FILES.len()
278        + 1;
279    info!("Verified {total} required data files");
280    Ok(())
281}
282
283#[cfg(feature = "maintenance")]
284fn verify_files_exist_and_populated(names: &[&str], directory: &Path, label: &str) -> anyhow::Result<()> {
285    let mut missing = Vec::new();
286    let mut stubs = Vec::new();
287
288    for name in names {
289        let path = directory.join(format!("{name}.json"));
290        if !path.is_file() {
291            missing.push(name.to_string());
292            continue;
293        }
294
295        if is_stub(&path) {
296            stubs.push(name.to_string());
297        }
298    }
299
300    if !missing.is_empty() || !stubs.is_empty() {
301        let mut msg = String::new();
302        if !missing.is_empty() {
303            msg.push_str(&format!("  Missing: {}\n", missing.join(", ")));
304        }
305        if !stubs.is_empty() {
306            msg.push_str(&format!("  Empty stubs: {}\n", stubs.join(", ")));
307        }
308        anyhow::bail!(
309            "{label} verification failed - {}/{} files not populated:\n{msg}",
310            missing.len() + stubs.len(),
311            names.len()
312        );
313    }
314    Ok(())
315}
316
317/// A file is an invalid empty stub if it contains only `{}` or `[]`.
318#[cfg(feature = "maintenance")]
319fn is_stub(path: &Path) -> bool {
320    match fs::read_to_string(path) {
321        Ok(content) => {
322            let trimmed = content.trim();
323            trimmed == "{}" || trimmed == "[]"
324        }
325        Err(_) => true,
326    }
327}
328
329#[cfg(all(test, feature = "maintenance"))]
330mod source_version_writer_tests {
331    use crate::source_versions::{AWS_CLI_SOURCE, CFN_LINT_SOURCE, RESOURCE_SCHEMA_SOURCE, SourceVersions};
332
333    const SYNC_ONLY_MANIFEST: &str = r#"{
334        "cfn_lint_version":"https://github.com/aws-cloudformation/cfn-lint@1.54.0",
335        "resource_schema_version":"https://github.com/aws-cloudformation/resource-provider-enhanced-schemas@2026-08-07T18:20:13Z"
336    }"#;
337
338    #[test]
339    fn catalog_generator_records_the_aws_cli_entry_and_keeps_sync_entries() {
340        let recorded = SourceVersions::from_json(SYNC_ONLY_MANIFEST)
341            .expect("manifest should parse")
342            .with_aws_cli_version(format!("{AWS_CLI_SOURCE}@2.36.43"))
343            .expect("catalog update should be valid");
344        assert_eq!(recorded.aws_cli_version.as_deref(), Some("https://github.com/aws/aws-cli@2.36.43"));
345        assert_eq!(recorded.cfn_lint_version, format!("{CFN_LINT_SOURCE}@1.54.0"));
346        assert_eq!(recorded.resource_schema_version, format!("{RESOURCE_SCHEMA_SOURCE}@2026-08-07T18:20:13Z"));
347
348        let error = SourceVersions::from_json(SYNC_ONLY_MANIFEST)
349            .expect("manifest should parse")
350            .with_aws_cli_version("2.36.43".to_string())
351            .expect_err("unqualified version must fail");
352        assert!(error.contains(AWS_CLI_SOURCE));
353    }
354}
data_source/lib.rs

data_source/
lib.rs
