use std::path::Path;
use url::Url;
use wax::walk::Entry as _;
use super::ParseOptions;
use crate::parsers::hcl::ast::{Hcl, InlineHint};
use crate::sources::VersionedSource;
use crate::{Error, Result};
pub async fn parse_dir(
path: impl AsRef<Path>,
opts: &ParseOptions,
) -> impl IntoIterator<Item = Result<VersionedSource>> {
use futures::future::join_all;
use itertools::Itertools;
let pattern = if opts.recursive { "**/*.tf" } else { "*.tf" };
let glob = wax::Glob::new(pattern).unwrap();
let walker = glob
.walk(path.as_ref())
.filter_map(std::result::Result::ok)
.filter(|entry| opts.hidden || !entry.to_candidate_path().as_ref().starts_with('.'))
.map(parse);
join_all(walker)
.await
.into_iter()
.flatten_ok()
.map(|res| res.and_then(std::convert::identity))
}
async fn parse(entry: wax::walk::GlobEntry) -> Result<Vec<Result<VersionedSource>>> {
use tokio::fs;
use crate::parsers::hcl as parser;
let content = fs::read_to_string(entry.path())
.await
.map_err(|err| Error::io(err, "Failed to read Terraform file"))?;
let (ast, ast_errors) = parser::parse(&content)?;
let file = entry.path().to_string_lossy().into_owned();
let mut sources: Vec<_> = ast_errors
.into_iter()
.map(|err| {
Err(Error::HclParsing {
file: file.clone(),
err,
})
})
.collect();
sources.extend(providers(&ast, &entry).map(Ok));
for module in ast.find_blocks("module") {
sources.extend(module_source(module, &entry).map(Ok));
sources.extend(inline_hint_sources(module, &entry));
}
Ok(sources)
}
fn providers<'a>(
ast: &'a Hcl<'a>,
entry: &'a wax::walk::GlobEntry,
) -> impl Iterator<Item = VersionedSource> + 'a {
use crate::parsers::hcl::ast::expr::Expression;
use crate::sources::data::{TerraformSource, TerraformSourceKind};
use crate::util::hcl;
let required = ast
.get_block("terraform", None)
.and_then(|terraform| terraform.get_block("required_providers", None));
required
.into_iter()
.flat_map(|required| required.attributes.iter())
.filter_map(|attr| attr.expr.as_object())
.filter_map(move |obj| {
let address = obj.get_by_name("source").and_then(Expression::as_string)?;
let version_req = obj.get_by_name("version").and_then(Expression::as_string)?;
let loc = hcl::compute_location(address, entry)
.with_span(version_req.span().start(), version_req.span().end());
let source = VersionedSource {
source: TerraformSource {
kind: TerraformSourceKind::Provider,
address: address.to_string(),
}
.into(),
current: Some(version_req.to_string()),
required: Some(version_req.to_string()),
};
Some(source.with_location(loc))
})
}
fn module_source(
module: &crate::parsers::hcl::ast::Block<'_>,
entry: &wax::walk::GlobEntry,
) -> Option<VersionedSource> {
use crate::sources::data::{TerraformSource, TerraformSourceKind};
use crate::util::hcl;
let address = module
.get_attribute("source")
.and_then(|attr| attr.expr.as_string())?;
if is_not_a_registry_address(&address) {
return None;
}
let version_req = module
.get_attribute("version")
.and_then(|attr| attr.expr.as_string());
let loc = hcl::compute_location_with_span(address, version_req, entry);
let source = VersionedSource {
source: TerraformSource {
kind: TerraformSourceKind::Module,
address: address.to_string(),
}
.into(),
current: version_req.map(|value| value.to_string()),
required: version_req.map(|value| value.to_string()),
};
Some(source.with_location(loc))
}
fn inline_hint_sources(
module: &crate::parsers::hcl::ast::Block<'_>,
entry: &wax::walk::GlobEntry,
) -> Vec<Result<VersionedSource>> {
use crate::sources::data::Source as BaseSource;
use crate::util::hcl;
let mut sources = Vec::new();
let hinted = module
.get_attributes_with_inline_hints()
.into_iter()
.filter_map(|attr| {
attr.hint
.as_ref()
.map(|InlineHint::Source(hint)| (attr, hint))
});
for (attribute, source_hint) in hinted {
let repository = source_hint.address;
let repo_string = hcl::convert_into_string(repository);
let repo_as_url = match Url::parse(repo_string.as_str()) {
Ok(url) => url,
Err(err) => {
sources.push(Err(Error::InvalidUrl {
repo_url: repo_string,
err,
}));
continue;
}
};
if is_not_a_registry_address(&repository) {
continue;
}
let version_constraint = source_hint.version_constraint.map(hcl::convert_into_string);
let version = attribute.clone().second_term_as_string();
let loc = hcl::compute_location_with_span(repository, version, entry);
let Some(current) = version else {
sources.push(Err(Error::invalid_inline_hint(
Error::invalid_source(
repo_as_url.clone(),
"the attribute following an inline hint must be a quoted version",
),
repository.to_string(),
)));
continue;
};
let source = VersionedSource {
source: match BaseSource::parse_url(&repo_as_url) {
Ok(source) => source.into(),
Err(err) => {
sources.push(Err(Error::invalid_inline_hint(err, repository.to_string())));
continue;
}
},
current: Some(current.to_string()),
required: version_constraint,
};
sources.push(Ok(source.with_location(loc)));
}
sources
}
fn is_not_a_registry_address(address: &str) -> bool {
address.starts_with("git::") || address.starts_with("./") || address.starts_with("../")
}
#[cfg(test)]
mod tests {
use itertools::Itertools;
use super::parse_dir;
use crate::sources::data::{HelmSource, Source as SourceData, TerraformSource};
use crate::sources::parse::Location;
use crate::sources::{VersionedSource, parse::ParseOptions};
const TEST_DIRECTORY: &str = "tests/sources/parse";
fn result_non_recursive() -> Vec<VersionedSource> {
vec![
VersionedSource::new(TerraformSource::new_provider("gitlabhq/gitlab"))
.with_current("~> 3.8")
.with_required("~> 3.8")
.with_location(Location::new("main.sample.test.tf", 4, 17).with_span(100, 108)),
VersionedSource::new(TerraformSource::new_module(
"gitlab.com/vigigloo/tf-modules/k8slimitednamespace",
))
.with_current("0.3.5")
.with_required("0.3.5")
.with_location(Location::new("main.sample.test.tf", 12, 18).with_span(271, 278)),
VersionedSource::new(TerraformSource::new_module(
"gitlab.com/vigigloo/tf-modules/k8slimitednamespace",
))
.with_location(Location::new("main.sample.test.tf", 17, 18)),
VersionedSource::new(TerraformSource::new_module(
"gitlab.com/vigigloo/tools-k8s/mattermost",
))
.with_current("0.1.0")
.with_required("0.1.0")
.with_location(Location::new("main.sample.test.tf", 21, 13).with_span(477, 484)),
VersionedSource::new(SourceData::Helm(HelmSource {
repository: "https://helm.nginx.com/stable".to_string(),
name: "nginx-ingress".to_string(),
}))
.with_required("> 0.10.0 , < 0.18.0")
.with_current("0.17.1")
.with_location(Location::new("main.sample.test.tf", 29, 18).with_span(772, 780)),
VersionedSource::new(TerraformSource::new_module(
"gitlab.com/vigigloo/tf-modules/k8slimitednamespace",
))
.with_current("2.0.0")
.with_required("2.0.0")
.with_location(Location::new("main.sample.test.tf", 33, 18).with_span(951, 958)),
VersionedSource::new(SourceData::Helm(HelmSource {
repository: "https://helm.nginx.com/stable".to_string(),
name: "nginx-ingress".to_string(),
}))
.with_current("0.17.0")
.with_required("0.17.0")
.with_location(Location::new("Chart.yaml", 6, 14).with_span(92, 100)),
]
}
#[tokio::test]
async fn parse_non_recursive() {
let (mut versions, errors): (Vec<_>, Vec<_>) = parse_dir(
TEST_DIRECTORY,
&ParseOptions {
recursive: false,
hidden: false,
},
)
.await
.into_iter()
.partition_result();
let mut wanted = result_non_recursive();
wanted.retain(|source| {
if let Some(loc) = source.source.location() {
!loc.path().to_string_lossy().ends_with("Chart.yaml")
} else {
true
}
});
wanted.sort_unstable();
versions.sort_unstable();
assert_eq!(versions, wanted);
assert_eq!(errors.len(), 1);
}
}