mod executable;
pub mod generator;
mod interface_library;
mod link_type;
mod misc;
mod object_library;
pub mod project;
mod starlark_api;
mod starlark_context;
mod starlark_executable;
mod starlark_fmt;
mod starlark_generator;
mod starlark_global;
mod starlark_interface_library;
mod starlark_link_target;
mod starlark_object_library;
mod starlark_project;
mod starlark_static_library;
mod static_library;
pub mod target;
pub mod toolchain;
use std::{
collections::{BTreeMap, HashMap},
fs,
path::{Path, PathBuf},
sync::{Arc, Mutex},
time::Duration,
};
use anyhow::anyhow;
use base64::Engine;
use flate2::read::GzDecoder;
use reqwest::StatusCode;
use serde::Deserialize;
use starlark::{
environment::{
Globals, GlobalsBuilder,
Module,
},
eval::Evaluator,
syntax::{
AstModule, Dialect,
DialectTypes,
},
};
use tar::Archive;
use project::Project;
use starlark_api::err_msg;
use starlark_global::{PkgOpt, StarGlobal};
use starlark_project::StarProject;
use toolchain::Toolchain;
type PkgOptMap = HashMap<String, HashMap<String, PkgOpt>>;
const CATAPULT_TOML: &str = "catapult.toml";
const BUILD_CATAPULT: &str = "build.catapult";
#[derive(Debug, Deserialize)]
struct Manifest {
package: PackageManifest,
dependencies: Option<BTreeMap<String, DependencyManifest>>,
options: Option<ManifestOptions>,
package_options: Option<HashMap<String, PkgOpt>>,
}
#[derive(Debug, Deserialize)]
struct PackageManifest {
name: String,
source: Option<String>,
}
#[derive(Debug, Deserialize)]
struct DependencyManifest {
version: Option<String>,
registry: Option<String>,
channel: Option<String>,
path: Option<String>,
git: Option<String>,
options: Option<HashMap<String, PkgOpt>>,
}
#[derive(Debug, Default, Deserialize)]
struct ManifestOptions {
c_standard: Option<String>,
cpp_standard: Option<String>,
position_independent_code: Option<bool>,
}
#[derive(Debug)]
pub struct GlobalOptions {
pub c_standard: Option<String>,
pub cpp_standard: Option<String>,
pub position_independent_code: Option<bool>,
}
fn read_manifest(src_dir: &Path) -> Result<Manifest, anyhow::Error> {
let manifest_path = src_dir.join(CATAPULT_TOML);
let catapult_toml = match fs::read_to_string(&manifest_path) {
Ok(x) => x,
Err(e) => return err_msg(format!("Error opening {}: {}", manifest_path.display(), e)),
};
let manifest = match toml::from_str::<Manifest>(&catapult_toml) {
Ok(x) => x,
Err(e) => return err_msg(format!("Error reading {}: {}", manifest_path.display(), e)),
};
Ok(manifest)
}
fn map_to_pkg_opt_map(opt_map: BTreeMap<String, BTreeMap<String, String>>) -> Result<PkgOptMap, anyhow::Error> {
type SerdeErr = toml::de::Error;
fn deserialize_pkg_opt(kv: (String, String)) -> Result<(String, PkgOpt), SerdeErr> {
let deserializer = toml::de::ValueDeserializer::new(&kv.1);
let opt_val = PkgOpt::deserialize(deserializer)?;
Ok((kv.0, opt_val))
}
match opt_map
.into_iter()
.map(|(k, im)| {
let val = im
.into_iter()
.map(deserialize_pkg_opt)
.collect::<Result<HashMap<String, PkgOpt>, _>>()?;
Ok((k, val))
})
.collect::<Result<PkgOptMap, SerdeErr>>()
{
Ok(x) => Ok(x),
Err(e) => Err(anyhow!(format!("Could not deserialize package option: {}", e))),
}
}
pub fn parse_project(
toolchain: &Toolchain,
package_options: BTreeMap<String, BTreeMap<String, String>>,
) -> Result<(Arc<Project>, GlobalOptions), anyhow::Error> {
let src_dir = PathBuf::from(".");
let manifest_options = read_manifest(&src_dir)?.options.unwrap_or_default();
let global_options = GlobalOptions {
c_standard: manifest_options.c_standard,
cpp_standard: manifest_options.cpp_standard,
position_independent_code: manifest_options.position_independent_code,
};
let mut combined_deps = BTreeMap::new();
let package_options = map_to_pkg_opt_map(package_options)?;
let project =
parse_project_inner(src_dir, &global_options, &package_options, HashMap::new(), toolchain, &mut combined_deps)?;
match project.into_project() {
Ok(x) => Ok((x, global_options)),
Err(e) => Err(anyhow!(e)),
}
}
#[derive(Deserialize)]
struct PackageRecord {
hash: String,
manifest: String,
recipe: String,
}
fn download_from_registry(
mut registry: String,
name: &str,
info_version: Option<String>,
info_channel: Option<String>,
) -> Result<PathBuf, anyhow::Error> {
let version = match &info_version {
Some(x) => x,
None => return Err(anyhow::anyhow!("Field \"version\" required for dependency \"{}\"", name)),
};
let channel = match &info_channel {
Some(x) => x,
None => return Err(anyhow::anyhow!("Field \"channel\" required for dependency \"{}\"", name)),
};
if !registry.ends_with('/') {
registry += "/";
}
let url = match reqwest::Url::parse(®istry) {
Ok(x) => x,
Err(e) => return Err(anyhow::anyhow!(e)),
};
let url = match url.join(&("get".to_owned() + "/" + name + "/" + version + "/" + channel)) {
Ok(x) => x,
Err(e) => return Err(anyhow::anyhow!(e)),
};
println!("Fetching dependency \"{}\" from {} ...", name, url);
let resp = match reqwest::blocking::Client::builder()
.build()?
.get(url.clone())
.timeout(Duration::from_secs(10))
.send()
{
Ok(resp) => resp,
Err(err) => return Err(anyhow!("Error trying to fetch \"{}\" from {}:\n {}", name, url, err)),
};
match resp.status() {
StatusCode::OK => (),
x => return Err(anyhow!("Request GET \"{}\" returned status {}", url, x)),
}
let resp_json = match resp.json::<PackageRecord>() {
Ok(x) => x,
Err(e) => return Err(anyhow!(e)),
};
let cache_dir = match dirs::cache_dir() {
Some(x) => x,
None => return Err(anyhow!("Could not find a HOME directory")),
};
let pkg_cache_path = cache_dir.join("catapult").join("cache").join(name).join(channel);
println!("pkg_cache_path: {:?}", pkg_cache_path);
let hash_path = pkg_cache_path.join("catapult.hash");
if let Ok(hash) = fs::read_to_string(&hash_path) {
if hash.trim() == resp_json.hash.trim() {
log::debug!("Package found in cache. It will not be downloaded: {name}");
return Ok(pkg_cache_path);
} else {
log::info!(
r#"A cached package was found but its hash does not match the one reported by the registry. It will be re-downloaded.
Package: {name}
On-disk hash: {}
Registry hash: {}"#,
hash.trim(),
resp_json.hash
);
}
}
let manifest_bytes = base64::engine::general_purpose::STANDARD_NO_PAD.decode(resp_json.manifest)?;
let manifest_str = std::str::from_utf8(&manifest_bytes)?;
let manifest = match toml::from_str::<Manifest>(manifest_str) {
Ok(x) => x,
Err(e) => return err_msg(format!("Error reading dependency manifest of {}: {}", name, e)),
};
let pkg_source_url = match manifest.package.source {
Some(x) => x,
None => return Err(anyhow!("Dependency manifest did not contain source. ({})", name)),
};
let src_data_resp = match reqwest::blocking::get(&pkg_source_url) {
Ok(resp) => resp,
Err(err) => panic!("Error: {}", err),
};
match src_data_resp.status() {
StatusCode::OK => (),
x => return Err(anyhow!("Request GET \"{}\" returned status {}", pkg_source_url, x)),
}
let tar = GzDecoder::new(src_data_resp);
let mut archive = Archive::new(tar);
archive.unpack(&pkg_cache_path)?;
let manifest_path = pkg_cache_path.join(CATAPULT_TOML);
match fs::write(manifest_path, manifest_bytes) {
Ok(x) => x,
Err(e) => return Err(anyhow!(e)),
};
let recipe_path = pkg_cache_path.join(BUILD_CATAPULT);
let recipe_bytes = base64::engine::general_purpose::STANDARD_NO_PAD.decode(resp_json.recipe)?;
match fs::write(recipe_path, recipe_bytes) {
Ok(x) => x,
Err(e) => return Err(anyhow!(e)),
};
match fs::write(hash_path, resp_json.hash.as_bytes()) {
Ok(x) => x,
Err(e) => return Err(anyhow!(e)),
}
Ok(pkg_cache_path)
}
fn parse_project_inner(
src_dir: PathBuf,
global_options: &GlobalOptions,
package_options: &PkgOptMap,
mut pkg_opt_underrides: HashMap<String, PkgOpt>,
toolchain: &Toolchain,
dep_map: &mut BTreeMap<String, Arc<StarProject>>,
) -> Result<StarProject, anyhow::Error> {
log::debug!("parse_project_inner {}", src_dir.display());
let manifest = read_manifest(&src_dir)?;
if let Some(pkg_opts) = package_options.get(&manifest.package.name) {
for (opt_name, opt_val) in pkg_opts {
pkg_opt_underrides.insert(opt_name.clone(), opt_val.clone());
}
}
let mut pkg_opts = package_options.clone();
pkg_opts.insert(manifest.package.name.clone(), pkg_opt_underrides);
let mut dependent_projects = Vec::new();
for (name, info) in manifest.dependencies.unwrap_or(BTreeMap::new()) {
if let Some(dep_proj) = dep_map.get(&name) {
dependent_projects.push(dep_proj.clone());
}
let pkg_opt_underrides = info.options.unwrap_or_default();
if let Some(registry) = info.registry {
let dep_path = download_from_registry(registry, &name, info.version, info.channel)?;
let dep_proj =
parse_project_inner(dep_path, global_options, &pkg_opts, pkg_opt_underrides, toolchain, dep_map)?;
let dep_proj = Arc::new(dep_proj);
dependent_projects.push(dep_proj.clone());
dep_map.insert(name, dep_proj);
} else if info.git.is_some() {
todo!();
} else if let Some(dep_path) = info.path {
let dep_proj = parse_project_inner(
PathBuf::from(&dep_path),
global_options,
&pkg_opts,
pkg_opt_underrides,
toolchain,
dep_map,
)?; let dep_proj = Arc::new(dep_proj);
dependent_projects.push(dep_proj.clone());
dep_map.insert(name, dep_proj);
} else {
return err_msg("Dependency must specify either \"registry\" or \"git\" or \"path\"".to_owned());
}
}
let mut option_overrides = manifest.package_options.unwrap_or_default();
if let Some(pkg_opts) = pkg_opts.get(&manifest.package.name) {
for (opt_name, opt_val) in pkg_opts {
log::debug!("Override option: {opt_name}");
if option_overrides.contains_key(opt_name) {
option_overrides.insert(opt_name.clone(), opt_val.clone());
} else {
log::error!("Package \"{}\" does not provide option \"{opt_name}\"", manifest.package.name);
}
}
}
let recipe_path = src_dir.join(BUILD_CATAPULT);
let starlark_code = match fs::read_to_string(&recipe_path) {
Ok(x) => x,
Err(e) => return err_msg(format!("Error reading \"{}\": {e}", recipe_path.display())),
};
let this_project = parse_module(
manifest.package.name.clone(),
dependent_projects,
global_options,
option_overrides,
toolchain,
src_dir,
starlark_code,
)?;
Ok(this_project)
}
pub(crate) fn setup(
project: &Arc<Mutex<StarProject>>,
global_options: &GlobalOptions,
package_options: HashMap<String, PkgOpt>,
toolchain: &Toolchain,
) -> Globals {
let mut globals_builder = GlobalsBuilder::standard();
starlark::environment::LibraryExtension::Print.add(&mut globals_builder);
globals_builder.set("GLOBAL", StarGlobal::new(global_options, package_options, toolchain));
starlark_api::build_api(project, &mut globals_builder);
globals_builder.build()
}
pub(crate) fn parse_module(
name: String,
deps: Vec<Arc<StarProject>>,
global_options: &GlobalOptions,
package_options: HashMap<String, PkgOpt>,
toolchain: &Toolchain,
current_dir: PathBuf,
starlark_code: String,
) -> Result<StarProject, anyhow::Error> {
let dialect = Dialect {
enable_types: DialectTypes::Enable,
enable_f_strings: true,
..Dialect::default()
};
let ast = match AstModule::parse(BUILD_CATAPULT, starlark_code, &dialect) {
Ok(x) => x,
Err(e) => panic!("AstModule::parse: {}", e),
};
let project_writable = Arc::new(Mutex::new(StarProject::new(name, current_dir, deps.clone())));
let module = Module::new();
for dep_proj in deps {
let proj_value = module.heap().alloc(StarProject::clone(&dep_proj));
module.set(&dep_proj.name, proj_value);
}
{
let mut eval = Evaluator::new(&module);
let globals = setup(&project_writable, global_options, package_options, toolchain);
eval.eval_module(ast, &globals).map_err(|e| e.into_anyhow())?;
}
let frozen_module = module.freeze()?;
let mut project = match project_writable.lock() {
Ok(x) => x.clone(),
Err(e) => return err_msg(format!("Could not lock project mutex: {e}")),
};
project.generator_names = frozen_module
.names()
.filter(|name| name.as_str().starts_with("__gen_"))
.map(|name| (name.as_str().to_string(), frozen_module.get(name.as_str()).unwrap()))
.collect();
Ok(project)
}