#![doc = include_str!("../README.md")]
#![warn(clippy::all, clippy::pedantic)]
#![allow(clippy::missing_errors_doc, clippy::cast_precision_loss)]
use std::env::home_dir;
use std::ffi::OsStr;
use std::fmt::{Display, Formatter};
use std::{
collections::BTreeMap,
env::{current_dir, set_current_dir},
fs::{DirEntry, File, create_dir_all},
io::Write,
path::Path,
process::Command,
sync::Arc,
};
struct PercentageDisplay(f64);
impl Display for PercentageDisplay {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
write!(f, "{:.2}%", self.0)
}
}
mod css;
mod generate;
mod js;
use crate::generate::action::generate_action_bindings;
use ordinary_config::{ActionLang, OrdinaryConfig};
use parking_lot::Mutex;
use swc_common::{FileName, FilePathMapping, SourceFile, SourceMap};
use swc_html_ast::Child;
use swc_html_codegen::writer::basic::BasicHtmlWriterConfig;
use swc_html_codegen::{Emit, writer::basic::BasicHtmlWriter};
use swc_html_minifier::{minify_document, option::MinifyOptions};
use swc_html_parser::parse_file_as_document;
use swc_html_parser::parser::ParserConfig;
const BASE_CLIENT: &str = include_str!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/static/client_template.rs"
));
const BASE_SERVER_TOML: &str = include_str!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/static/ServerTemplate.toml"
));
const BASE_CLIENT_TOML: &str = include_str!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/static/ClientTemplate.toml"
));
const BASE_ACTION_TOML: &str = include_str!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/static/ServerActionGen.toml"
));
const APPEND_WASM_JS: &str = include_str!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/static/append_wasm.js"
));
const JS_ONLY_JS: &str = include_str!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/static/javascript_only.js"
));
const CORE_JS: &str = include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/static/core.js"));
#[allow(clippy::too_many_lines, clippy::missing_panics_doc)]
pub fn build(path: &str, no_cache: bool) -> Result<(), Box<dyn std::error::Error>> {
let wasm_opt_path = home_dir()
.expect("home dir doesn't exist")
.join(".ordinary")
.join("bin")
.join("wasm-opt");
if !wasm_opt_path.exists() {
tracing::warn!(
"wasm-opt not installed at {} (built WASM modules will not be further optimized) - for install, see: https://codeberg.org/ordinarylabs/Ordinary/src/branch/main/scripts/install/binaryen",
wasm_opt_path.display()
);
}
let start_dir = current_dir()?;
let path = Path::new(path);
set_current_dir(path)?;
let project_dir = current_dir()?;
let config_str = std::fs::read_to_string("./ordinary.json")?;
let config: OrdinaryConfig = serde_json::from_str(&config_str)?;
let gen_dir_path = project_dir.join(".ordinary/gen");
let client_dir_path = gen_dir_path.join("client");
let server_dir_path = gen_dir_path.join("server");
let templates_dir_path = gen_dir_path.join("templates");
let hashes_dir_path = gen_dir_path.join("hashes");
create_dir_all(&client_dir_path)?;
create_dir_all(&server_dir_path)?;
create_dir_all(&templates_dir_path)?;
create_dir_all(&hashes_dir_path)?;
let mut content_def_map = BTreeMap::new();
if let Some(content) = config.content {
for content_def in content.definitions {
content_def_map.insert(content_def.name.clone(), content_def.clone());
}
}
let mut model_config_map = BTreeMap::new();
if let Some(models) = config.models {
for model_config in models {
model_config_map.insert(model_config.name.clone(), model_config.clone());
}
}
let mut integration_config_map = BTreeMap::new();
if let Some(integrations) = config.integrations {
for integration_config in integrations {
integration_config_map
.insert(integration_config.name.clone(), integration_config.clone());
}
}
if let Some(actions) = config.actions.clone() {
for action_config in actions {
let cache_dir = gen_dir_path
.join("actions")
.join("cache")
.join(&action_config.name);
let input_action_dir = Path::new(&action_config.dir_path);
create_dir_all(&cache_dir)?;
let should_build = Arc::new(Mutex::new(false));
traverse(input_action_dir, &|entry| {
let path = entry.path();
if let Some(str_path) = path.to_str()
&& (str_path.contains(".vscode")
|| str_path.contains("target")
|| str_path.contains("node_modules"))
{
return Ok(());
}
let curr_content = std::fs::read(&path).unwrap_or_else(|_| Vec::new());
if let Ok(child_path) = path.strip_prefix(input_action_dir) {
if let Some(parent) = child_path.parent()
&& let Err(err) = create_dir_all(cache_dir.join(parent))
{
tracing::error!(%err, "failed to create dir");
}
let cache_path = cache_dir.join(child_path);
let cached_content = std::fs::read(&cache_path).unwrap_or_else(|_| Vec::new());
if curr_content != cached_content
&& let Ok(mut content_file) = File::create(&cache_path)
&& content_file.write_all(&curr_content).is_ok()
{
let mut should_build = should_build.lock();
*should_build = true;
}
}
Ok(())
})?;
let should_build = should_build.lock();
if *should_build || no_cache {
let (generated_code, extras) = generate_action_bindings(
&action_config,
&model_config_map,
&content_def_map,
&integration_config_map,
&config.auth,
&config.domain,
)?;
if let Some((main_rs, cargo_toml, type_defs)) = extras {
let path = project_dir.join(&action_config.dir_path);
create_dir_all(path.join("src"))?;
let mut cargo_file = File::create(path.join("Cargo.toml"))?;
cargo_file.write_all(cargo_toml.as_bytes())?;
let mut main_file = File::create(path.join("src").join("main.rs"))?;
main_file.write_all(main_rs.as_bytes())?;
match action_config.lang {
ActionLang::Rust => {}
ActionLang::JavaScript => {
let mut type_file = File::create(path.join("index.d.ts"))?;
type_file.write_all(type_defs.as_bytes())?;
let curr_dir = current_dir()?;
set_current_dir(path)?;
Command::new("pnpm").args(["install"]).output()?;
Command::new("pnpm").args(["run", "build"]).output()?;
set_current_dir(curr_dir)?;
}
}
}
let action_dir_path = gen_dir_path.join("actions").join(&action_config.name);
create_dir_all(action_dir_path.join("src"))?;
let cargo = action_dir_path.join("Cargo.toml");
let mut cargo_file = File::create(cargo)?;
cargo_file.write_all(BASE_ACTION_TOML.as_bytes())?;
let lib = action_dir_path.join("src").join("lib.rs");
let mut lib_file = File::create(lib)?;
lib_file.write_all(generated_code.as_bytes())?;
let path = project_dir.join(action_config.dir_path);
set_current_dir(path)?;
let output = Command::new("cargo")
.args(["build", "--release", "--target", "wasm32-wasip1"])
.output()?;
if output.status.success() {
let action_path = "target/wasm32-wasip1/release/action.wasm";
if let Some(wasm_opt) = action_config.wasm_opt
&& wasm_opt_path.exists()
{
let opt_output = Command::new(&wasm_opt_path)
.args([
"--all-features",
action_path,
"-o",
action_path,
wasm_opt.as_flag(),
])
.output()?;
if !opt_output.status.success() {
print!("{}", std::str::from_utf8(&opt_output.stderr)?);
}
}
let action_bytes = std::fs::read(action_path)?;
tracing::info!(
name = action_config.name,
language = ?action_config.lang,
size.bin = %bytesize::ByteSize(action_bytes.len() as u64).display().si(),
"action"
);
} else {
tracing::error!(
"failed for action '{}'\n\n{}",
action_config.name,
String::from_utf8_lossy(&output.stderr)
);
}
set_current_dir(&project_dir)?;
} else {
tracing::info!(
name = action_config.name,
"action has not changed; skipping."
);
}
}
}
let mut client_file = BASE_CLIENT.to_string();
if let Some(templates) = config.templates {
for template_config in templates {
if let Some(template_path) = &template_config.path {
let path = project_dir.join(template_path);
let mut template_string = std::fs::read_to_string(&path)?;
template_string = template_string.replace("{{ version }}", &config.version);
let (template_final, csp_hashes) = if template_config.minify == Some(true) {
if template_config.mime == "text/html"
|| template_config.mime == "text/html; charset=utf-8"
{
swc_common::GLOBALS.set(&swc_common::Globals::new(), || {
let cm = SourceMap::new(FilePathMapping::empty());
let fm =
cm.new_source_file(FileName::Anon.into(), template_string.clone());
let mut errors = Vec::new();
if let Ok(mut document) =
parse_file_as_document(&fm, ParserConfig::default(), &mut errors)
{
minify_document(
&mut document,
&MinifyOptions {
collapse_whitespaces:
swc_html_minifier::option::CollapseWhitespaces::Smart,
..Default::default()
},
);
let mut html_string = String::new();
let wr = BasicHtmlWriter::new(
&mut html_string,
None,
BasicHtmlWriterConfig::default(),
);
let mut generator = swc_html_codegen::CodeGenerator::new(
wr,
swc_html_codegen::CodegenConfig {
minify: true,
..Default::default()
},
);
if let Err(err) = generator.emit(&document) {
tracing::error!(%err, "failed to generate html");
}
let cm = SourceMap::new(FilePathMapping::empty());
let fm =
cm.new_source_file(FileName::Anon.into(), html_string.clone());
let csp_hashes = save_inline_hashes(
&mut errors,
&fm,
&hashes_dir_path,
&template_config.name,
);
(
html_string.as_bytes().to_vec(),
csp_hashes.has_any().then_some(csp_hashes),
)
} else {
tracing::error!("failed to parse file as HTML");
(vec![], None)
}
})
} else {
(template_string.as_bytes().to_vec(), None)
}
} else if template_config.mime == "text/html"
|| template_config.mime == "text/html; charset=utf-8"
{
let cm = SourceMap::new(FilePathMapping::empty());
let fm = cm.new_source_file(FileName::Anon.into(), template_string.clone());
let mut errors = Vec::new();
let csp_hashes = save_inline_hashes(
&mut errors,
&fm,
&hashes_dir_path,
&template_config.name,
);
(
template_string.as_bytes().to_vec(),
csp_hashes.has_any().then_some(csp_hashes),
)
} else {
(template_string.as_bytes().to_vec(), None)
};
let file_name = match path.extension() {
Some(ext) => match ext.to_str() {
Some(ext) => &format!("{}.{}", &template_config.name, ext),
None => &template_config.name,
},
None => &template_config.name,
};
let mut global_vars = BTreeMap::new();
if let Some(globals) = config.globals.clone() {
for global_var in globals {
global_vars.insert(global_var.name, global_var.value);
}
}
let (server, client) = generate::template::generate_template_renderers(
&config.domain,
&config.version,
file_name,
template_config.clone(),
&content_def_map,
&model_config_map,
&global_vars,
&config.error,
&config.auth,
);
client_file = format!("{client_file}\n{client}");
let file_path = templates_dir_path.join(file_name);
if !no_cache && file_path.exists() && std::fs::read(&file_path)? == template_final {
tracing::info!(
name = template_config.name,
"template has not changed; skipping."
);
continue;
}
let mut file = File::create(file_path)?;
file.write_all(&template_final)?;
create_dir_all(server_dir_path.join(&template_config.name).join("src"))?;
let server_main_rs = server_dir_path
.join(&template_config.name)
.join("src")
.join("main.rs");
let mut server_main_rs_file = File::create(server_main_rs)?;
server_main_rs_file.write_all(server.as_bytes())?;
let server_cargo = server_dir_path
.join(&template_config.name)
.join("Cargo.toml");
let mut server_cargo_file = File::create(server_cargo)?;
server_cargo_file.write_all(BASE_SERVER_TOML.as_bytes())?;
let server_askama = server_dir_path
.join(&template_config.name)
.join("askama.toml");
let mut server_askama_file = File::create(server_askama)?;
server_askama_file.write_all(
r#"# generated
[general]
dirs = ["../../templates"]
[[escaper]]
path = "askama::filters::Text"
extensions = ["js", "json"]
"#
.as_bytes(),
)?;
let path = server_dir_path.join(&template_config.name);
set_current_dir(path)?;
Command::new("cargo").args(["fmt"]).output()?;
let output = Command::new("cargo")
.args(["build", "--release", "--target", "wasm32-wasip1"])
.output()?;
if output.status.success() {
let template_wasm_path = "target/wasm32-wasip1/release/template.wasm";
if let Some(wasm_opt) = template_config.wasm_opt
&& wasm_opt_path.exists()
{
let opt_output = Command::new(&wasm_opt_path)
.args([
"--all-features",
template_wasm_path,
"-o",
template_wasm_path,
wasm_opt.as_flag(),
])
.output()?;
if !opt_output.status.success() {
print!("{}", std::str::from_utf8(&opt_output.stderr)?);
}
}
let template_bytes = std::fs::read(template_wasm_path)?;
if template_config.minify == Some(true) {
tracing::info!(
name = template_config.name,
mime = template_config.mime,
size.bin = %bytesize::ByteSize(template_bytes.len() as u64)
.display()
.si(),
size.source = %bytesize::ByteSize(template_string.len() as u64)
.display()
.si(),
size.minified = %bytesize::ByteSize(template_final.len() as u64)
.display()
.si(),
size.reduction = %PercentageDisplay(((template_string.len() as f64 - template_final.len() as f64)
/ template_string.len() as f64)
* 100.0),
csp = csp_hashes.map(display),
"template"
);
} else {
tracing::info!(
name = template_config.name,
mime = template_config.mime,
size.bin = %bytesize::ByteSize(template_bytes.len() as u64)
.display()
.si(),
size.source = %bytesize::ByteSize(template_string.len() as u64)
.display()
.si(),
csp = csp_hashes.map(display),
"template"
);
}
} else {
tracing::error!(
name = template_config.name,
"failed for template\n{}",
String::from_utf8_lossy(&output.stderr)
);
}
}
set_current_dir(&project_dir)?;
}
}
if config.auth.is_some()
|| config.client_e2ee == Some(true)
|| config.client_rendering == Some(true)
{
create_dir_all(client_dir_path.join("src"))?;
let client_lib_rs = client_dir_path.join("src").join("lib.rs");
let mut client_lib_file = File::create(client_lib_rs)?;
client_lib_file.write_all(client_file.as_bytes())?;
let client_cargo = client_dir_path.join("Cargo.toml");
let mut client_cargo_file = File::create(client_cargo)?;
client_cargo_file.write_all(BASE_CLIENT_TOML.as_bytes())?;
let client_askama = client_dir_path.join("askama.toml");
let mut client_askama_file = File::create(client_askama)?;
client_askama_file.write_all(
r#"# generated
[general]
dirs = ["../templates"]
[[escaper]]
path = "askama::filters::Text"
extensions = ["js", "json"]
"#
.as_bytes(),
)?;
set_current_dir("./.ordinary/gen/client")?;
Command::new("cargo").args(["fmt"]).output()?;
let build_output = Command::new("cargo")
.args([
"build",
"--release",
"--lib",
"--target",
"wasm32-unknown-unknown",
])
.output()?;
if !build_output.status.success() {
print!("{}", std::str::from_utf8(&build_output.stderr)?);
}
wasm_bindgen_cli::wasm_bindgen::run_cli_with_args([
"wasm-bindgen",
"target/wasm32-unknown-unknown/release/client.wasm",
"--out-dir",
"wasm",
"--typescript",
"--target",
"web",
])?;
if wasm_opt_path.exists() {
let opt_output = Command::new(wasm_opt_path)
.args([
"--all-features",
"wasm/client_bg.wasm",
"-o",
"wasm/client_bg_opt.wasm",
"-O4",
])
.output()?;
if !opt_output.status.success() {
print!("{}", std::str::from_utf8(&opt_output.stderr)?);
}
}
}
set_current_dir(&project_dir)?;
if config.auth.is_some() || config.client_events == Some(true) {
create_dir_all("./.ordinary/gen/client/js")?;
let mut core_js_file = File::create("./.ordinary/gen/client/js/core.js")?;
core_js_file.write_all(js::minify(CORE_JS)?.as_bytes())?;
}
if config.auth.is_some()
|| config.client_e2ee == Some(true)
|| config.client_rendering == Some(true)
{
let client_js = std::fs::read_to_string("./.ordinary/gen/client/wasm/client.js")?;
if !client_js.contains(&js::minify(APPEND_WASM_JS)?) {
let final_js = format!("{client_js}\n{APPEND_WASM_JS}");
let mut client_js_file = File::create("./.ordinary/gen/client/wasm/client.js")?;
client_js_file.write_all(js::minify(&final_js)?.as_bytes())?;
}
}
if config.auth.is_some() {
let mut js_only_client = File::create("./.ordinary/gen/client/js/client.js")?;
js_only_client.write_all(js::minify(JS_ONLY_JS)?.as_bytes())?;
}
if let Some(assets) = config.assets
&& (assets.minify_css == Some(true) || assets.minify_js == Some(true))
{
let dir_path = Path::new(&assets.dir_path);
let gen_path = Path::new("./.ordinary/gen/assets");
create_dir_all(gen_path)?;
traverse(dir_path, &|entry| {
let path = entry.path();
let path2 = entry.path();
let rel_path = path2.strip_prefix(dir_path)?;
if path.extension() == Some(OsStr::new("css")) {
let file_str = std::fs::read_to_string(path)?;
let file_str_len = file_str.len();
if let Ok(minified) = css::minify(&file_str) {
let dest_path = gen_path.join(rel_path);
if let Some(parent) = gen_path.join(rel_path).parent() {
create_dir_all(parent)?;
let mut content_file = File::create(&dest_path)?;
content_file.write_all(minified.as_bytes())?;
tracing::info!(
path = %rel_path.display(),
ext = "css",
size.source = %bytesize::ByteSize(file_str_len as u64).display().si(),
size.minified = %bytesize::ByteSize(minified.len() as u64).display().si(),
size.reduction = %PercentageDisplay(((file_str_len as f64 - minified.len() as f64) / file_str_len as f64)
* 100.0)
,
"asset"
);
}
}
} else if path.extension() == Some(OsStr::new("js")) {
let file_str = std::fs::read_to_string(path)?;
let file_str_len = file_str.len();
if let Ok(minified) = js::minify(&file_str) {
let dest_path = gen_path.join(rel_path);
if let Some(parent) = gen_path.join(rel_path).parent() {
create_dir_all(parent)?;
let mut content_file = File::create(&dest_path)?;
content_file.write_all(minified.as_bytes())?;
tracing::info!(
path = %rel_path.display(),
ext= "js",
size.source = %bytesize::ByteSize(file_str_len as u64).display().si(),
size.minified = %bytesize::ByteSize(minified.len() as u64).display().si(),
size.reduction = %PercentageDisplay(((file_str_len as f64 - minified.len() as f64) / file_str_len as f64)
* 100.0)
,
"asset"
);
}
}
}
Ok(())
})?;
}
set_current_dir(start_dir)?;
Ok(())
}
fn save_inline_hashes(
errors: &mut Vec<Error>,
fm: &Arc<SourceFile>,
dir: &Path,
name: &str,
) -> CspValues {
let new_dir_path = dir.join(name);
if let Err(err) = create_dir_all(&new_dir_path) {
tracing::error!(%err, "failed to create dir");
}
let mut csp_values = CspValues {
script_src_inline_hashes: vec![],
style_src_inline_hashes: vec![],
};
if let Ok(document) = parse_file_as_document(fm, ParserConfig::default(), errors) {
walk_document(document.children.clone(), &mut csp_values);
if let Ok(script_src_json) = serde_json::to_string(&csp_values.script_src_inline_hashes)
&& let Err(err) = std::fs::write(
new_dir_path.join("script-src.json"),
script_src_json.as_bytes(),
)
{
tracing::error!(%err, "failed to write file");
}
if let Ok(style_src_json) = serde_json::to_string(&csp_values.style_src_inline_hashes)
&& let Err(err) = std::fs::write(
new_dir_path.join("style-src.json"),
style_src_json.as_bytes(),
)
{
tracing::error!(%err, "failed to write file");
}
}
csp_values
}
#[allow(clippy::type_complexity)]
pub fn traverse(
dir: &Path,
cb: &dyn Fn(&DirEntry) -> Result<(), Box<dyn std::error::Error>>,
) -> Result<(), Box<dyn std::error::Error>> {
if dir.is_dir() {
for entry in std::fs::read_dir(dir)? {
let entry = entry?;
let path = entry.path();
if path.is_dir() {
traverse(&path, cb)?;
} else {
cb(&entry)?;
}
}
}
Ok(())
}
use base64::{Engine as B64Engine, engine::general_purpose::STANDARD as b64};
use sha2::{Digest, Sha256};
use swc_html_parser::error::Error;
struct CspValues {
script_src_inline_hashes: Vec<String>,
style_src_inline_hashes: Vec<String>,
}
impl CspValues {
fn has_any(&self) -> bool {
if self.script_src_inline_hashes.is_empty() {
!self.style_src_inline_hashes.is_empty()
} else {
true
}
}
}
impl Display for CspValues {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
if !self.style_src_inline_hashes.is_empty() {
write!(f, "style-src")?;
for hash in &self.style_src_inline_hashes {
write!(f, " '{hash}'")?;
}
if !self.script_src_inline_hashes.is_empty() {
write!(f, "; ")?;
}
}
if !self.script_src_inline_hashes.is_empty() {
write!(f, "script-src")?;
for hash in &self.script_src_inline_hashes {
write!(f, " '{hash}'")?;
}
}
write!(f, "")
}
}
#[allow(clippy::similar_names)]
fn walk_document(children: Vec<Child>, hashes: &mut CspValues) {
for child in children {
if let Some(element) = child.element() {
if element.tag_name == "script" {
let mut is_inline = true;
for attr in element.attributes {
if attr.name == "src" {
is_inline = false;
}
}
if is_inline && let Some(Child::Text(text)) = element.children.first() {
let mut hasher = Sha256::new();
hasher.update(text.data.as_bytes());
let hash = hasher.finalize().to_vec();
let mut b64_hash = b64.encode(hash);
b64_hash.insert_str(0, "sha256-");
hashes.script_src_inline_hashes.push(b64_hash);
}
} else if element.tag_name == "style" {
if let Some(Child::Text(text)) = element.children.first() {
let mut hasher = Sha256::new();
hasher.update(text.data.as_bytes());
let hash = hasher.finalize().to_vec();
let mut b64_hash = b64.encode(hash);
b64_hash.insert_str(0, "sha256-");
hashes.style_src_inline_hashes.push(b64_hash);
}
} else {
walk_document(element.children, hashes);
}
}
}
}