#![doc = include_str!("../Documentation.md")]
#![forbid(unsafe_code)]
use kcode_k1_transaction_id::TxId;
use kcode_k1_web_bootstrap_archive::{Archive, Package};
use kcode_k1_web_bootstrap_preflight::inspect;
use kcode_k1_web_coding::{CheckOutcome, K1WebCoding, WebCodingConfig, WebCodingRevisions};
use kcode_k1_web_package::{
AuthorityId, DependencySelector, SourceFile, SourcePackage, WebFamily, WebId,
};
use kcode_k1_web_podman::WebPodmanConfig;
use kcode_k1_web_projection::{K1WebProjection, PublishStatus};
use semver::Version;
use serde::Serialize;
use sha2::{Digest, Sha256};
use std::collections::BTreeSet;
use std::fs::{File, OpenOptions};
use std::io::Write;
use std::path::{Path, PathBuf};
use std::sync::Arc;
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct ImporterRevisions {
pub boot: String,
pub schema: String,
pub route: String,
pub harness: String,
pub check_policy: String,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct ImportedPackage {
name: String,
version: Version,
source_sha256: [u8; 32],
}
impl ImportedPackage {
pub fn name(&self) -> &str {
&self.name
}
pub fn version(&self) -> &Version {
&self.version
}
pub const fn source_sha256(&self) -> [u8; 32] {
self.source_sha256
}
}
#[derive(Clone)]
pub struct WebBootstrapImporter {
inner: Arc<Inner>,
}
struct Inner {
cache_root: PathBuf,
projection_root: PathBuf,
revisions: ImporterRevisions,
podman: WebPodmanConfig,
projection: Arc<K1WebProjection>,
}
impl WebBootstrapImporter {
pub fn new(
cache_root: impl Into<PathBuf>,
projection_root: impl Into<PathBuf>,
revisions: ImporterRevisions,
podman: WebPodmanConfig,
projection: Arc<K1WebProjection>,
) -> Result<Self, String> {
let cache_root = cache_root.into();
let projection_root = projection_root.into();
require_directory(&cache_root, "Web bootstrap cache")?;
require_directory(&projection_root, "Web projection")?;
if [
&revisions.boot,
&revisions.schema,
&revisions.route,
&revisions.harness,
&revisions.check_policy,
]
.iter()
.any(|value| value.is_empty())
{
return Err("Web bootstrap importer revisions must be nonempty".to_owned());
}
Ok(Self {
inner: Arc::new(Inner {
cache_root,
projection_root,
revisions,
podman,
projection,
}),
})
}
pub fn import_all(
&self,
archive: &Archive,
authority: [u8; 12],
user: [u8; 12],
log_path: impl AsRef<Path>,
) -> Result<Vec<ImportedPackage>, String> {
let log_path = log_path.as_ref();
let mut log = ImportLog::append(log_path)?;
let plan = inspect(archive);
log.line(&format!(
"IMPORT START packages={} warnings={}",
plan.packages().len(),
plan.warnings().len()
))?;
let authority = AuthorityId::new(TxId::from_bytes(authority));
let mut prepared = std::iter::repeat_with(|| None)
.take(archive.packages().len())
.collect::<Vec<Option<PreparedPackage>>>();
let mut permanent_failures = Vec::new();
for &index in plan.order() {
let package = &archive.packages()[index];
let label = package_label(package);
match prepare_package(package, authority) {
Ok(item) => prepared[index] = Some(item),
Err(error) => {
log.line(&format!("ERROR prepare {label}: {error}"))?;
permanent_failures.push(index);
}
}
}
let revisions = &self.inner.revisions;
let config = match WebCodingConfig::new(
WebCodingRevisions {
boot: revisions.boot.clone(),
schema: revisions.schema.clone(),
route: revisions.route.clone(),
harness: revisions.harness.clone(),
check_policy: revisions.check_policy.clone(),
},
self.inner.projection_root.clone(),
self.inner.podman.clone(),
) {
Ok(config) => config,
Err(error) => {
log.line(&format!("ERROR configure Web bootstrap coding: {error}"))?;
log.line(&format!(
"FAILURE imported=0 remaining={}",
archive.packages().len()
))?;
return Err(format!(
"configure Web bootstrap coding: {error}; inspect {}",
log_path.display()
));
}
};
let mut coding = match K1WebCoding::open(
&self.inner.cache_root,
TxId::from_bytes(user),
config,
Arc::clone(&self.inner.projection),
) {
Ok(coding) => coding,
Err(error) => {
log.line(&format!("ERROR open Web bootstrap coding: {error}"))?;
log.line(&format!(
"FAILURE imported=0 remaining={}",
archive.packages().len()
))?;
return Err(format!(
"open Web bootstrap coding: {error}; inspect {}",
log_path.display()
));
}
};
let authorize = |family: &WebFamily| Ok(family.authority() == authority);
let mut pending = plan
.order()
.iter()
.copied()
.filter(|index| prepared[*index].is_some())
.collect::<Vec<_>>();
let mut imported = Vec::with_capacity(pending.len());
let mut round = 0_usize;
while !pending.is_empty() {
round += 1;
log.line(&format!("ROUND {round} pending={}", pending.len()))?;
let mut next = Vec::new();
let mut progress = 0_usize;
for index in std::mem::take(&mut pending) {
let item = prepared[index]
.as_ref()
.expect("pending packages were prepared");
let label = coordinate_label(&item.coordinate);
log.line(&format!("ATTEMPT {label}"))?;
match self.import_one(&mut coding, item, &authorize) {
Ok(()) => {
log.line(&format!("IMPORTED {label}"))?;
imported.push(ImportedPackage {
name: item.coordinate.name.clone(),
version: item.coordinate.version.clone(),
source_sha256: item.source_sha256,
});
progress += 1;
}
Err(error) => {
log.line(&format!("ERROR import {label}: {error}"))?;
next.push(index);
}
}
}
if next.is_empty() {
break;
}
if progress == 0 {
pending = next;
break;
}
for &index in &next {
let item = prepared[index]
.as_ref()
.expect("retry packages were prepared");
log.line(&format!(
"RETRY deferred {} after round {round} progress",
coordinate_label(&item.coordinate)
))?;
}
pending = next;
}
let remaining = permanent_failures.len() + pending.len();
if remaining != 0 {
log.line(&format!(
"FAILURE imported={} remaining={remaining}",
imported.len()
))?;
return Err(format!(
"Web bootstrap left {remaining} package(s) concretely unimported; inspect {}",
log_path.display()
));
}
log.line(&format!("SUCCESS packages={}", imported.len()))?;
Ok(imported)
}
fn import_one(
&self,
coding: &mut K1WebCoding,
item: &PreparedPackage,
authorize: &impl Fn(&WebFamily) -> Result<bool, String>,
) -> Result<(), String> {
let label = coordinate_label(&item.coordinate);
coding
.write(&item.source, authorize)
.map_err(|error| format!("stage package: {error}"))?;
let check = coding
.check_fresh(item.source.id(), authorize)
.map_err(|error| format!("fresh Chromium check: {error}"))?;
if !matches!(check, CheckOutcome::Checked(_)) {
return Err("fresh Chromium check was unexpectedly reused".to_owned());
}
match self
.inner
.projection
.load(item.source.id())
.map_err(|error| format!("inspect existing publication: {error}"))?
{
Some(existing) if existing.as_ref() == &item.source => Ok(()),
Some(_) => Err(format!(
"published Web package {label} has conflicting source bytes"
)),
None => {
let completion = coding
.publish(item.source.id(), authorize, &|_, _| Ok(true))
.map_err(|error| format!("publish package: {error}"))?;
if completion.outcome.status == PublishStatus::Conflict {
Err(format!(
"published Web package {label} reported an immutable conflict"
))
} else {
Ok(())
}
}
}
}
}
pub fn write_preflight_log(archive: &Archive, log_path: impl AsRef<Path>) -> Result<(), String> {
let log_path = log_path.as_ref();
let plan = inspect(archive);
let mut log = ImportLog::replace(log_path)?;
log.line(&format!(
"START Web bootstrap preflight packages={}",
plan.packages().len()
))?;
for warning in plan.warnings() {
log.line(warning)?;
}
log.line(&format!(
"PREFLIGHT COMPLETE packages={} warnings={}",
plan.packages().len(),
plan.warnings().len()
))
}
struct ImportLog(File);
impl ImportLog {
fn replace(path: &Path) -> Result<Self, String> {
OpenOptions::new()
.create(true)
.truncate(true)
.write(true)
.open(path)
.map(Self)
.map_err(|error| {
format!(
"initialize Web bootstrap import log {}: {error}",
path.display()
)
})
}
fn append(path: &Path) -> Result<Self, String> {
OpenOptions::new()
.create(true)
.append(true)
.open(path)
.map(Self)
.map_err(|error| format!("open Web bootstrap import log {}: {error}", path.display()))
}
fn line(&mut self, line: &str) -> Result<(), String> {
writeln!(self.0, "{line}")
.map_err(|error| format!("write Web bootstrap import log: {error}"))?;
self.0
.flush()
.map_err(|error| format!("flush Web bootstrap import log: {error}"))
}
}
#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)]
struct Coordinate {
name: String,
version: Version,
}
#[derive(Debug)]
struct PreparedPackage {
coordinate: Coordinate,
source_sha256: [u8; 32],
source: SourcePackage,
}
#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd, Serialize)]
struct GeneratedDependency {
authority: String,
name: String,
selector: String,
}
#[derive(Serialize)]
struct Header<'a> {
dependencies: &'a [GeneratedDependency],
}
#[derive(Serialize)]
struct Manifest<'a> {
name: &'a str,
version: String,
entry: &'a str,
tests: &'a str,
dependencies: &'a [GeneratedDependency],
}
fn prepare_package(package: &Package, authority: AuthorityId) -> Result<PreparedPackage, String> {
let coordinate = Coordinate {
name: package.name().to_owned(),
version: package.version().clone(),
};
let family = WebFamily::new(authority, coordinate.name.clone()).map_err(|error| {
format!(
"assign Web package {} authority: {error}",
coordinate_label(&coordinate)
)
})?;
let id = WebId::new(family, coordinate.version.clone()).map_err(|error| {
format!(
"assign Web package {} version: {error}",
coordinate_label(&coordinate)
)
})?;
let authority_text = authority.to_string();
let (dependencies, rewrites) = usable_dependencies(package, authority, &coordinate);
let original_documentation = package
.files()
.iter()
.find(|file| file.path() == "Documentation.md")
.ok_or_else(|| {
format!(
"Web bootstrap package {} lacks Documentation.md",
coordinate_label(&coordinate)
)
})?
.bytes();
let header = serde_json::to_string(&Header {
dependencies: &dependencies,
})
.map_err(|error| {
format!(
"encode Web dependencies for {}: {error}",
coordinate_label(&coordinate)
)
})?;
let mut documentation = format!("<!-- k1-web/v1\n{header}\n-->\n").into_bytes();
documentation.extend_from_slice(original_documentation);
let manifest = serde_json::to_vec(&Manifest {
name: package.name(),
version: package.version().to_string(),
entry: package.entry(),
tests: package.tests(),
dependencies: &dependencies,
})
.map_err(|error| {
format!(
"encode Web manifest for {}: {error}",
coordinate_label(&coordinate)
)
})?;
let mut files = Vec::with_capacity(package.files().len() + 1);
for file in package.files() {
if matches!(file.path(), "Documentation.md" | "k1-web.json") {
continue;
}
files.push(SourceFile::new(
file.path(),
rewrite_file(file.path(), file.bytes(), &authority_text, &rewrites),
));
}
files.push(SourceFile::new("Documentation.md", documentation));
files.push(SourceFile::new("k1-web.json", manifest));
let source = SourcePackage::new(id, files).map_err(|error| {
format!(
"construct Web bootstrap package {}: {error}",
coordinate_label(&coordinate)
)
})?;
Ok(PreparedPackage {
coordinate,
source_sha256: source_digest(&source),
source,
})
}
fn usable_dependencies(
package: &Package,
authority: AuthorityId,
coordinate: &Coordinate,
) -> (Vec<GeneratedDependency>, BTreeSet<(String, String)>) {
let authority_text = authority.to_string();
let mut dependencies = BTreeSet::new();
let mut rewrites = BTreeSet::new();
for dependency in package.dependencies() {
if WebFamily::new(authority, dependency.name().to_owned()).is_err() {
continue;
}
let Ok(selector) = DependencySelector::parse(dependency.selector()) else {
continue;
};
rewrites.insert((
dependency.name().to_owned(),
dependency.selector().to_owned(),
));
if dependency.name() == coordinate.name && selector.matches(&coordinate.version) {
continue;
}
dependencies.insert(GeneratedDependency {
authority: authority_text.clone(),
name: dependency.name().to_owned(),
selector: dependency.selector().to_owned(),
});
}
(dependencies.into_iter().collect(), rewrites)
}
fn rewrite_file(
path: &str,
bytes: &[u8],
authority: &str,
declarations: &BTreeSet<(String, String)>,
) -> Vec<u8> {
if !matches!(extension(path), "js" | "mjs" | "html" | "css") {
return bytes.to_vec();
}
let Ok(source) = std::str::from_utf8(bytes) else {
return bytes.to_vec();
};
let mut rewritten = source.to_owned();
for (name, selector) in declarations {
let legacy = format!("/module/{name}/v{selector}/");
let route_selector = if selector == "*" {
"*".to_owned()
} else {
format!("v{selector}")
};
let current = format!("/{authority}/{name}/{route_selector}/");
rewritten = rewritten.replace(&legacy, ¤t);
}
rewritten.into_bytes()
}
fn extension(path: &str) -> &str {
path.rsplit_once('.').map_or("", |(_, extension)| extension)
}
fn source_digest(source: &SourcePackage) -> [u8; 32] {
let mut hash = Sha256::new();
for file in source.files() {
hash.update((file.path().len() as u64).to_le_bytes());
hash.update(file.path().as_bytes());
hash.update((file.bytes().len() as u64).to_le_bytes());
hash.update(file.bytes());
}
hash.finalize().into()
}
fn package_label(package: &Package) -> String {
format!("{} {}", package.name(), package.version())
}
fn coordinate_label(coordinate: &Coordinate) -> String {
format!("{} {}", coordinate.name, coordinate.version)
}
fn require_directory(path: &Path, label: &str) -> Result<(), String> {
let metadata = std::fs::symlink_metadata(path)
.map_err(|error| format!("inspect {label} {}: {error}", path.display()))?;
if metadata.is_dir() && !metadata.file_type().is_symlink() {
Ok(())
} else {
Err(format!("{label} is not an ordinary directory"))
}
}
#[cfg(test)]
mod tests {
use super::*;
use kcode_k1_web_bootstrap_archive::read;
use serde_json::Value;
use std::fs::File;
use std::io::Write;
use zip::write::SimpleFileOptions;
fn archive(root: &Path, name: &str, manifest: &str, entries: &[(&str, &[u8])]) -> Archive {
let path = root.join(name);
let file = File::create(&path).unwrap();
let mut zip = zip::ZipWriter::new(file);
zip.start_file("manifest.json", SimpleFileOptions::default())
.unwrap();
zip.write_all(manifest.as_bytes()).unwrap();
for (path, bytes) in entries {
zip.start_file(*path, SimpleFileOptions::default()).unwrap();
zip.write_all(bytes).unwrap();
}
zip.finish().unwrap();
read(&path).unwrap().archive
}
fn file<'a>(source: &'a SourcePackage, path: &str) -> &'a [u8] {
source
.files()
.iter()
.find(|file| file.path() == path)
.unwrap()
.bytes()
}
fn prepare_all(archive: &Archive, authority: AuthorityId) -> Vec<PreparedPackage> {
let plan = inspect(archive);
plan.order()
.iter()
.map(|index| prepare_package(&archive.packages()[*index], authority).unwrap())
.collect()
}
#[test]
fn prepares_complete_trees_current_routes_and_exact_metadata() {
let root = tempfile::tempdir().unwrap();
let manifest = r#"{"schema":1,"packages":[{"name":"dep","version":"1.5.0","entry":"index.js","tests":"tests.js","dependencies":[]},{"name":"dep","version":"2.1.0","entry":"index.js","tests":"tests.js","dependencies":[]},{"name":"bridge","version":"1.0.0","entry":"index.js","tests":"tests.js","dependencies":[{"name":"dep","selector":"2"}]},{"name":"app","version":"1.0.0","entry":"src/index.js","tests":"tests.js","dependencies":[{"name":"app","selector":"1"},{"name":"dep","selector":"1"}]}]}"#;
let entries: Vec<(&str, &[u8])> = vec![
(
"packages/dep/1.5.0/Documentation.md",
b"dep one docs\r\n",
),
(
"packages/dep/1.5.0/index.js",
b"export const version=1;\n",
),
(
"packages/dep/1.5.0/tests.js",
b"export function runTests(){}\n",
),
(
"packages/dep/1.5.0/assets/deep/image.bin",
&[0, 1, 255],
),
(
"packages/dep/2.1.0/Documentation.md",
b"dep two docs\n",
),
(
"packages/dep/2.1.0/index.js",
b"export const version=2;\n",
),
(
"packages/dep/2.1.0/tests.js",
b"export function runTests(){}\n",
),
(
"packages/bridge/1.0.0/Documentation.md",
b"bridge docs\n",
),
(
"packages/bridge/1.0.0/index.js",
b"import '/module/dep/v2/index.js';\nexport {};\n",
),
(
"packages/bridge/1.0.0/tests.js",
b"export function runTests(){}\n",
),
(
"packages/app/1.0.0/Documentation.md",
b"app docs\r\n",
),
(
"packages/app/1.0.0/src/index.js",
b"import '/module/dep/v1/index.js';\nexport const self='/module/app/v1/src/index.js';\n",
),
(
"packages/app/1.0.0/tests.js",
b"export function runTests(){}\n",
),
(
"packages/app/1.0.0/styles/main.css",
b"@import '/module/dep/v1/style.css';\n",
),
];
let archive = archive(root.path(), "complete.zip", manifest, &entries);
let authority = AuthorityId::new(TxId::from_bytes([1; 12]));
let authority_text = authority.to_string();
let prepared = prepare_all(&archive, authority);
let dep_one = prepared
.iter()
.find(|item| {
item.coordinate.name == "dep" && item.coordinate.version == Version::new(1, 5, 0)
})
.unwrap();
assert_eq!(file(&dep_one.source, "assets/deep/image.bin"), [0, 1, 255]);
let app = prepared
.iter()
.find(|item| item.coordinate.name == "app")
.unwrap();
assert_eq!(app.source.entry(), "src/index.js");
assert_eq!(app.source.tests(), "tests.js");
assert_eq!(app.source.dependencies().len(), 1);
assert_eq!(app.source.dependencies()[0].name(), "dep");
let expected_documentation = format!(
"<!-- k1-web/v1\n{{\"dependencies\":[{{\"authority\":\"{authority_text}\",\"name\":\"dep\",\"selector\":\"1\"}}]}}\n-->\napp docs\r\n"
);
assert_eq!(
file(&app.source, "Documentation.md"),
expected_documentation.as_bytes()
);
let generated: Value = serde_json::from_slice(file(&app.source, "k1-web.json")).unwrap();
assert_eq!(generated["name"], "app");
assert_eq!(generated["version"], "1.0.0");
assert_eq!(generated["entry"], "src/index.js");
assert_eq!(generated["tests"], "tests.js");
assert_eq!(generated["dependencies"].as_array().unwrap().len(), 1);
let code = std::str::from_utf8(file(&app.source, "src/index.js")).unwrap();
assert!(code.contains(&format!("/{authority_text}/dep/v1/index.js")));
assert!(code.contains(&format!("/{authority_text}/app/v1/src/index.js")));
assert!(!code.contains("/module/"));
let style = std::str::from_utf8(file(&app.source, "styles/main.css")).unwrap();
assert!(style.contains(&format!("/{authority_text}/dep/v1/style.css")));
}
#[test]
fn warnings_do_not_prevent_preparation() {
let root = tempfile::tempdir().unwrap();
let manifest = r#"{"schema":1,"packages":[{"name":"a","version":"1.0.0","entry":"index.js","tests":"tests.js","dependencies":[{"name":"missing","selector":"1"}]},{"name":"b","version":"1.0.0","entry":"index.js","tests":"tests.js","dependencies":[{"name":"a","selector":"1"},{"name":"bad name","selector":"not a selector"}]},{"name":"c","version":"1.0.0","entry":"index.js","tests":"tests.js","dependencies":[{"name":"b","selector":"1"},{"name":"c","selector":"1"}]}]}"#;
let entries: Vec<(&str, &[u8])> = vec![
("packages/a/1.0.0/Documentation.md", b"a docs\n"),
(
"packages/a/1.0.0/index.js",
b"import '/module/missing/v1/index.js';\n",
),
(
"packages/a/1.0.0/tests.js",
b"export function runTests(){}\n",
),
("packages/b/1.0.0/Documentation.md", b"b docs\n"),
(
"packages/b/1.0.0/index.js",
b"import '/module/a/v1/index.js';\n",
),
(
"packages/b/1.0.0/tests.js",
b"export function runTests(){}\n",
),
("packages/c/1.0.0/Documentation.md", b"c docs\n"),
(
"packages/c/1.0.0/index.js",
b"import '/module/b/v1/index.js';\n",
),
(
"packages/c/1.0.0/tests.js",
b"export function runTests(){}\n",
),
];
let archive = archive(root.path(), "warnings.zip", manifest, &entries);
let plan = inspect(&archive);
assert!(!plan.warnings().is_empty());
let authority = AuthorityId::new(TxId::from_bytes([2; 12]));
let prepared = prepare_all(&archive, authority);
assert_eq!(prepared.len(), 3);
let a = prepared
.iter()
.find(|item| item.coordinate.name == "a")
.unwrap();
assert_eq!(a.source.dependencies().len(), 1);
assert_eq!(a.source.dependencies()[0].name(), "missing");
let b = prepared
.iter()
.find(|item| item.coordinate.name == "b")
.unwrap();
assert_eq!(b.source.dependencies().len(), 1);
assert_eq!(b.source.dependencies()[0].name(), "a");
let c = prepared
.iter()
.find(|item| item.coordinate.name == "c")
.unwrap();
assert_eq!(c.source.dependencies().len(), 1);
assert_eq!(c.source.dependencies()[0].name(), "b");
}
#[test]
fn preflight_log_contains_all_warnings_and_is_replaced() {
let root = tempfile::tempdir().unwrap();
let manifest = r#"{"schema":1,"packages":[{"name":"a","version":"1.0.0","entry":"index.js","tests":"tests.js","dependencies":[{"name":"missing-one","selector":"1"},{"name":"missing-two","selector":"2"}]}]}"#;
let entries: Vec<(&str, &[u8])> = vec![
("packages/a/1.0.0/Documentation.md", b"a docs\n"),
("packages/a/1.0.0/index.js", b"export {};\n"),
(
"packages/a/1.0.0/tests.js",
b"export function runTests(){}\n",
),
];
let archive = archive(root.path(), "log.zip", manifest, &entries);
let log = root.path().join("import.log");
std::fs::write(&log, "old contents\n").unwrap();
write_preflight_log(&archive, &log).unwrap();
let text = std::fs::read_to_string(log).unwrap();
assert!(!text.contains("old contents"));
assert_eq!(text.matches("WARN unresolved dependency").count(), 2);
assert!(text.contains("PREFLIGHT COMPLETE"));
}
}