#![doc = include_str!("../Documentation.md")]
#![forbid(unsafe_code)]
use kcode_k1_transaction_id::TxId;
use kcode_k1_web_bootstrap_archive::{Archive, Package};
use kcode_k1_web_coding::{CheckOutcome, K1WebCoding, WebCodingConfig, WebCodingRevisions};
use kcode_k1_web_package::{
AuthorityId, DependencySelector, SourceFile, SourcePackage, WebFamily, WebId,
};
use kcode_k1_web_podman::WebPodmanConfig;
use kcode_k1_web_projection::{K1WebProjection, PublishStatus};
use semver::Version;
use serde::Serialize;
use sha2::{Digest, Sha256};
use std::collections::{BTreeMap, BTreeSet};
use std::path::{Path, PathBuf};
use std::sync::Arc;
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct ImporterRevisions {
pub boot: String,
pub schema: String,
pub route: String,
pub harness: String,
pub check_policy: String,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct ImportedPackage {
name: String,
version: Version,
source_sha256: [u8; 32],
}
impl ImportedPackage {
pub fn name(&self) -> &str {
&self.name
}
pub fn version(&self) -> &Version {
&self.version
}
pub const fn source_sha256(&self) -> [u8; 32] {
self.source_sha256
}
}
#[derive(Clone)]
pub struct WebBootstrapImporter {
inner: Arc<Inner>,
}
struct Inner {
cache_root: PathBuf,
projection_root: PathBuf,
revisions: ImporterRevisions,
podman: WebPodmanConfig,
projection: Arc<K1WebProjection>,
}
impl WebBootstrapImporter {
pub fn new(
cache_root: impl Into<PathBuf>,
projection_root: impl Into<PathBuf>,
revisions: ImporterRevisions,
podman: WebPodmanConfig,
projection: Arc<K1WebProjection>,
) -> Result<Self, String> {
let cache_root = cache_root.into();
let projection_root = projection_root.into();
require_directory(&cache_root, "Web bootstrap cache")?;
require_directory(&projection_root, "Web projection")?;
if [
&revisions.boot,
&revisions.schema,
&revisions.route,
&revisions.harness,
&revisions.check_policy,
]
.iter()
.any(|value| value.is_empty())
{
return Err("Web bootstrap importer revisions must be nonempty".to_owned());
}
Ok(Self {
inner: Arc::new(Inner {
cache_root,
projection_root,
revisions,
podman,
projection,
}),
})
}
pub fn import_all(
&self,
archive: &Archive,
authority: [u8; 12],
user: [u8; 12],
) -> Result<Vec<ImportedPackage>, String> {
let authority = AuthorityId::new(TxId::from_bytes(authority));
let prepared = prepare_all(archive, authority)?;
let revisions = &self.inner.revisions;
let config = WebCodingConfig::new(
WebCodingRevisions {
boot: revisions.boot.clone(),
schema: revisions.schema.clone(),
route: revisions.route.clone(),
harness: revisions.harness.clone(),
check_policy: revisions.check_policy.clone(),
},
self.inner.projection_root.clone(),
self.inner.podman.clone(),
)
.map_err(|error| format!("configure Web bootstrap coding: {error}"))?;
let mut coding = K1WebCoding::open(
&self.inner.cache_root,
TxId::from_bytes(user),
config,
Arc::clone(&self.inner.projection),
)
.map_err(|error| format!("open Web bootstrap coding: {error}"))?;
let authorize = |family: &WebFamily| Ok(family.authority() == authority);
let mut imported = Vec::with_capacity(prepared.len());
for item in prepared {
let label = format!("{} {}", item.coordinate.name, item.coordinate.version);
coding
.write(&item.source, &authorize)
.map_err(|error| format!("stage Web bootstrap package {label}: {error}"))?;
let check = coding
.check_fresh(item.source.id(), &authorize)
.map_err(|error| format!("check Web bootstrap package {label}: {error}"))?;
if !matches!(check, CheckOutcome::Checked(_)) {
return Err(format!("fresh Web check was reused for {label}"));
}
match self
.inner
.projection
.load(item.source.id())
.map_err(|error| format!("inspect published Web package {label}: {error}"))?
{
Some(existing) if existing.as_ref() == &item.source => {}
Some(_) => {
return Err(format!(
"published Web package {label} differs from bootstrap source"
));
}
None => {
let completion = coding
.publish(item.source.id(), &authorize, &|_, _| Ok(true))
.map_err(|error| {
format!("publish Web bootstrap package {label}: {error}")
})?;
if completion.outcome.status == PublishStatus::Conflict {
return Err(format!(
"published Web package {label} conflicts with bootstrap source"
));
}
}
}
imported.push(ImportedPackage {
name: item.coordinate.name,
version: item.coordinate.version,
source_sha256: item.source_sha256,
});
}
Ok(imported)
}
}
#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)]
struct Coordinate {
name: String,
version: Version,
}
struct PlannedPackage {
index: usize,
dependencies: Vec<GeneratedDependency>,
edges: BTreeSet<Coordinate>,
}
#[derive(Debug)]
struct PreparedPackage {
coordinate: Coordinate,
source_sha256: [u8; 32],
source: SourcePackage,
}
#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd, Serialize)]
struct GeneratedDependency {
authority: String,
name: String,
selector: String,
}
#[derive(Serialize)]
struct Header<'a> {
dependencies: &'a [GeneratedDependency],
}
#[derive(Serialize)]
struct Manifest<'a> {
name: &'a str,
version: String,
entry: &'a str,
tests: &'a str,
dependencies: &'a [GeneratedDependency],
}
fn prepare_all(archive: &Archive, authority: AuthorityId) -> Result<Vec<PreparedPackage>, String> {
let mut coordinates = BTreeMap::new();
for (index, package) in archive.packages().iter().enumerate() {
let coordinate = Coordinate {
name: package.name().to_owned(),
version: package.version().clone(),
};
if coordinates.insert(coordinate.clone(), index).is_some() {
return Err(format!(
"duplicate Web bootstrap package {} {}",
coordinate.name, coordinate.version
));
}
}
let authority_text = authority.to_string();
let mut plans = BTreeMap::new();
for (coordinate, index) in &coordinates {
let package = &archive.packages()[*index];
let mut dependencies = Vec::new();
let mut edges = BTreeSet::new();
for dependency in package.dependencies() {
WebFamily::new(authority, dependency.name().to_owned()).map_err(|error| {
format!(
"invalid dependency {} from {} {}: {error}",
dependency.name(),
coordinate.name,
coordinate.version
)
})?;
let selector = DependencySelector::parse(dependency.selector()).map_err(|error| {
format!(
"invalid selector {} for dependency {} from {} {}: {error}",
dependency.selector(),
dependency.name(),
coordinate.name,
coordinate.version
)
})?;
if dependency.name() == coordinate.name && selector.matches(&coordinate.version) {
continue;
}
let selected = coordinates
.keys()
.filter(|candidate| {
candidate.name == dependency.name() && selector.matches(&candidate.version)
})
.max_by(|left, right| left.version.cmp(&right.version))
.cloned()
.ok_or_else(|| {
format!(
"unresolved Web bootstrap dependency {} {} from {} {}",
dependency.name(),
dependency.selector(),
coordinate.name,
coordinate.version
)
})?;
edges.insert(selected);
dependencies.push(GeneratedDependency {
authority: authority_text.clone(),
name: dependency.name().to_owned(),
selector: dependency.selector().to_owned(),
});
}
dependencies.sort();
plans.insert(
coordinate.clone(),
PlannedPackage {
index: *index,
dependencies,
edges,
},
);
}
let order = dependency_order(&plans)?;
let mut prepared = Vec::with_capacity(order.len());
for coordinate in order {
let plan = plans
.get(&coordinate)
.expect("dependency order contains only planned coordinates");
let package = &archive.packages()[plan.index];
let source = prepare_package(package, authority, &plan.dependencies)?;
prepared.push(PreparedPackage {
coordinate,
source_sha256: source_digest(&source),
source,
});
}
Ok(prepared)
}
fn dependency_order(
plans: &BTreeMap<Coordinate, PlannedPackage>,
) -> Result<Vec<Coordinate>, String> {
fn visit(
coordinate: &Coordinate,
plans: &BTreeMap<Coordinate, PlannedPackage>,
states: &mut BTreeMap<Coordinate, u8>,
stack: &mut Vec<Coordinate>,
order: &mut Vec<Coordinate>,
) -> Result<(), String> {
match states.get(coordinate).copied() {
Some(2) => return Ok(()),
Some(1) => {
let start = stack
.iter()
.position(|item| item == coordinate)
.unwrap_or(0);
let mut cycle = stack[start..].to_vec();
cycle.push(coordinate.clone());
let text = cycle
.iter()
.map(|item| format!("{} {}", item.name, item.version))
.collect::<Vec<_>>()
.join(" -> ");
return Err(format!("non-self Web bootstrap dependency cycle: {text}"));
}
_ => {}
}
states.insert(coordinate.clone(), 1);
stack.push(coordinate.clone());
for dependency in &plans
.get(coordinate)
.expect("visited coordinate is planned")
.edges
{
visit(dependency, plans, states, stack, order)?;
}
let popped = stack.pop();
debug_assert_eq!(popped.as_ref(), Some(coordinate));
states.insert(coordinate.clone(), 2);
order.push(coordinate.clone());
Ok(())
}
let mut states = BTreeMap::new();
let mut stack = Vec::new();
let mut order = Vec::with_capacity(plans.len());
for coordinate in plans.keys() {
visit(coordinate, plans, &mut states, &mut stack, &mut order)?;
}
Ok(order)
}
fn prepare_package(
package: &Package,
authority: AuthorityId,
dependencies: &[GeneratedDependency],
) -> Result<SourcePackage, String> {
let family = WebFamily::new(authority, package.name().to_owned())
.map_err(|error| format!("assign Web package {} authority: {error}", package.name()))?;
let id = WebId::new(family, package.version().clone())
.map_err(|error| format!("assign Web package {} version: {error}", package.name()))?;
let authority_text = authority.to_string();
let original_documentation = package
.files()
.iter()
.find(|file| file.path() == "Documentation.md")
.ok_or_else(|| {
format!(
"Web bootstrap package {} {} lacks Documentation.md",
package.name(),
package.version()
)
})?
.bytes();
let header = serde_json::to_string(&Header { dependencies }).map_err(|error| {
format!(
"encode Web dependencies for {} {}: {error}",
package.name(),
package.version()
)
})?;
let mut documentation = format!("<!-- k1-web/v1\n{header}\n-->\n").into_bytes();
documentation.extend_from_slice(original_documentation);
let manifest = serde_json::to_vec(&Manifest {
name: package.name(),
version: package.version().to_string(),
entry: package.entry(),
tests: package.tests(),
dependencies,
})
.map_err(|error| {
format!(
"encode Web manifest for {} {}: {error}",
package.name(),
package.version()
)
})?;
let mut files = Vec::with_capacity(package.files().len() + 1);
for file in package.files() {
if matches!(file.path(), "Documentation.md" | "k1-web.json") {
continue;
}
files.push(SourceFile::new(
file.path(),
rewrite_file(
file.path(),
file.bytes(),
&authority_text,
package.dependencies(),
),
));
}
files.push(SourceFile::new("Documentation.md", documentation));
files.push(SourceFile::new("k1-web.json", manifest));
SourcePackage::new(id, files).map_err(|error| {
format!(
"construct Web bootstrap package {} {}: {error}",
package.name(),
package.version()
)
})
}
fn rewrite_file(
path: &str,
bytes: &[u8],
authority: &str,
dependencies: &[kcode_k1_web_bootstrap_archive::Dependency],
) -> Vec<u8> {
if !matches!(extension(path), "js" | "mjs" | "html" | "css") {
return bytes.to_vec();
}
let Ok(source) = std::str::from_utf8(bytes) else {
return bytes.to_vec();
};
let mut rewritten = source.to_owned();
let declarations = dependencies
.iter()
.map(|dependency| {
(
dependency.name().to_owned(),
dependency.selector().to_owned(),
)
})
.collect::<BTreeSet<_>>();
for (name, selector) in declarations {
let legacy = format!("/module/{name}/v{selector}/");
let route_selector = if selector == "*" {
"*".to_owned()
} else {
format!("v{selector}")
};
let current = format!("/{authority}/{name}/{route_selector}/");
rewritten = rewritten.replace(&legacy, ¤t);
}
rewritten.into_bytes()
}
fn extension(path: &str) -> &str {
path.rsplit_once('.').map_or("", |(_, extension)| extension)
}
fn source_digest(source: &SourcePackage) -> [u8; 32] {
let mut hash = Sha256::new();
for file in source.files() {
hash.update((file.path().len() as u64).to_le_bytes());
hash.update(file.path().as_bytes());
hash.update((file.bytes().len() as u64).to_le_bytes());
hash.update(file.bytes());
}
hash.finalize().into()
}
fn require_directory(path: &Path, label: &str) -> Result<(), String> {
let metadata = std::fs::symlink_metadata(path)
.map_err(|error| format!("inspect {label} {}: {error}", path.display()))?;
if metadata.is_dir() && !metadata.file_type().is_symlink() {
Ok(())
} else {
Err(format!("{label} is not an ordinary directory"))
}
}
#[cfg(test)]
mod tests {
use super::*;
use kcode_k1_web_bootstrap_archive::read;
use serde_json::Value;
use std::fs::File;
use std::io::Write;
use zip::write::SimpleFileOptions;
fn archive(root: &Path, name: &str, manifest: &str, entries: &[(&str, &[u8])]) -> Archive {
let path = root.join(name);
let file = File::create(&path).unwrap();
let mut zip = zip::ZipWriter::new(file);
zip.start_file("manifest.json", SimpleFileOptions::default())
.unwrap();
zip.write_all(manifest.as_bytes()).unwrap();
for (path, bytes) in entries {
zip.start_file(*path, SimpleFileOptions::default()).unwrap();
zip.write_all(bytes).unwrap();
}
zip.finish().unwrap();
read(&path).unwrap().archive
}
fn file<'a>(source: &'a SourcePackage, path: &str) -> &'a [u8] {
source
.files()
.iter()
.find(|file| file.path() == path)
.unwrap()
.bytes()
}
#[test]
fn prepares_complete_trees_current_routes_and_exact_metadata() {
let root = tempfile::tempdir().unwrap();
let manifest = r#"{"schema":1,"packages":[{"name":"dep","version":"1.5.0","entry":"index.js","tests":"tests.js","dependencies":[]},{"name":"dep","version":"2.1.0","entry":"index.js","tests":"tests.js","dependencies":[]},{"name":"bridge","version":"1.0.0","entry":"index.js","tests":"tests.js","dependencies":[{"name":"dep","selector":"2"}]},{"name":"app","version":"1.0.0","entry":"src/index.js","tests":"tests.js","dependencies":[{"name":"app","selector":"1"},{"name":"dep","selector":"1"}]}]}"#;
let entries: Vec<(&str, &[u8])> = vec![
(
"packages/dep/1.5.0/Documentation.md",
b"dep one docs\r\n",
),
(
"packages/dep/1.5.0/index.js",
b"export const version=1;\n",
),
(
"packages/dep/1.5.0/tests.js",
b"export function runTests(){}\n",
),
(
"packages/dep/1.5.0/assets/deep/image.bin",
&[0, 1, 255],
),
(
"packages/dep/2.1.0/Documentation.md",
b"dep two docs\n",
),
(
"packages/dep/2.1.0/index.js",
b"export const version=2;\n",
),
(
"packages/dep/2.1.0/tests.js",
b"export function runTests(){}\n",
),
(
"packages/bridge/1.0.0/Documentation.md",
b"bridge docs\n",
),
(
"packages/bridge/1.0.0/index.js",
b"import '/module/dep/v2/index.js';\nexport {};\n",
),
(
"packages/bridge/1.0.0/tests.js",
b"export function runTests(){}\n",
),
(
"packages/app/1.0.0/Documentation.md",
b"app docs\r\n",
),
(
"packages/app/1.0.0/src/index.js",
b"import '/module/dep/v1/index.js';\nexport const self='/module/app/v1/src/index.js';\n",
),
(
"packages/app/1.0.0/tests.js",
b"export function runTests(){}\n",
),
(
"packages/app/1.0.0/styles/main.css",
b"@import '/module/dep/v1/style.css';\n",
),
];
let archive = archive(root.path(), "complete.zip", manifest, &entries);
let authority = AuthorityId::new(TxId::from_bytes([1; 12]));
let authority_text = authority.to_string();
let prepared = prepare_all(&archive, authority).unwrap();
let dep_one = prepared
.iter()
.find(|item| {
item.coordinate.name == "dep" && item.coordinate.version == Version::new(1, 5, 0)
})
.unwrap();
assert_eq!(file(&dep_one.source, "assets/deep/image.bin"), [0, 1, 255]);
let app = prepared
.iter()
.find(|item| item.coordinate.name == "app")
.unwrap();
assert_eq!(app.source.entry(), "src/index.js");
assert_eq!(app.source.tests(), "tests.js");
assert_eq!(app.source.dependencies().len(), 1);
assert_eq!(app.source.dependencies()[0].name(), "dep");
let expected_documentation = format!(
"<!-- k1-web/v1\n{{\"dependencies\":[{{\"authority\":\"{authority_text}\",\"name\":\"dep\",\"selector\":\"1\"}}]}}\n-->\napp docs\r\n"
);
assert_eq!(
file(&app.source, "Documentation.md"),
expected_documentation.as_bytes()
);
let generated: Value = serde_json::from_slice(file(&app.source, "k1-web.json")).unwrap();
assert_eq!(generated["name"], "app");
assert_eq!(generated["version"], "1.0.0");
assert_eq!(generated["entry"], "src/index.js");
assert_eq!(generated["tests"], "tests.js");
assert_eq!(generated["dependencies"].as_array().unwrap().len(), 1);
let code = std::str::from_utf8(file(&app.source, "src/index.js")).unwrap();
assert!(code.contains(&format!("/{authority_text}/dep/v1/index.js")));
assert!(code.contains(&format!("/{authority_text}/app/v1/src/index.js")));
assert!(!code.contains("/module/"));
assert!(!code.contains(&format!("/module/{authority_text}")));
let style = std::str::from_utf8(file(&app.source, "styles/main.css")).unwrap();
assert!(style.contains(&format!("/{authority_text}/dep/v1/style.css")));
let dep_one_position = prepared
.iter()
.position(|item| {
item.coordinate.name == "dep" && item.coordinate.version == Version::new(1, 5, 0)
})
.unwrap();
let app_position = prepared
.iter()
.position(|item| item.coordinate.name == "app")
.unwrap();
let dep_two_position = prepared
.iter()
.position(|item| {
item.coordinate.name == "dep" && item.coordinate.version == Version::new(2, 1, 0)
})
.unwrap();
let bridge_position = prepared
.iter()
.position(|item| item.coordinate.name == "bridge")
.unwrap();
assert!(dep_one_position < app_position);
assert!(dep_two_position < bridge_position);
}
#[test]
fn rejects_unresolved_dependencies_and_non_self_cycles() {
let root = tempfile::tempdir().unwrap();
let unresolved_manifest = r#"{"schema":1,"packages":[{"name":"app","version":"1.0.0","entry":"index.js","tests":"tests.js","dependencies":[{"name":"missing","selector":"1"}]}]}"#;
let app_entries: Vec<(&str, &[u8])> = vec![
("packages/app/1.0.0/Documentation.md", b"app docs\n"),
("packages/app/1.0.0/index.js", b"export {};\n"),
(
"packages/app/1.0.0/tests.js",
b"export function runTests(){}\n",
),
];
let unresolved = archive(
root.path(),
"unresolved.zip",
unresolved_manifest,
&app_entries,
);
let authority = AuthorityId::new(TxId::from_bytes([2; 12]));
assert!(
prepare_all(&unresolved, authority)
.unwrap_err()
.contains("unresolved Web bootstrap dependency")
);
let cycle_manifest = r#"{"schema":1,"packages":[{"name":"a","version":"1.0.0","entry":"index.js","tests":"tests.js","dependencies":[{"name":"b","selector":"1"}]},{"name":"b","version":"1.0.0","entry":"index.js","tests":"tests.js","dependencies":[{"name":"a","selector":"1"}]}]}"#;
let cycle_entries: Vec<(&str, &[u8])> = vec![
("packages/a/1.0.0/Documentation.md", b"a docs\n"),
("packages/a/1.0.0/index.js", b"export {};\n"),
(
"packages/a/1.0.0/tests.js",
b"export function runTests(){}\n",
),
("packages/b/1.0.0/Documentation.md", b"b docs\n"),
("packages/b/1.0.0/index.js", b"export {};\n"),
(
"packages/b/1.0.0/tests.js",
b"export function runTests(){}\n",
),
];
let cycle = archive(root.path(), "cycle.zip", cycle_manifest, &cycle_entries);
assert!(
prepare_all(&cycle, authority)
.unwrap_err()
.contains("non-self Web bootstrap dependency cycle")
);
}
}