kcode_k1_web_bootstrap_import/

lib.rs

#![doc = include_str!("../Documentation.md")]
#![forbid(unsafe_code)]

use kcode_k1_transaction_id::TxId;
use kcode_k1_web_bootstrap_archive::{Archive, Package};
use kcode_k1_web_coding::{CheckOutcome, K1WebCoding, WebCodingConfig, WebCodingRevisions};
use kcode_k1_web_package::{
    AuthorityId, DependencySelector, SourceFile, SourcePackage, WebFamily, WebId,
};
use kcode_k1_web_podman::WebPodmanConfig;
use kcode_k1_web_projection::{K1WebProjection, PublishStatus};
use semver::Version;
use serde::Serialize;
use sha2::{Digest, Sha256};
use std::collections::{BTreeMap, BTreeSet};
use std::path::{Path, PathBuf};
use std::sync::Arc;

#[derive(Clone, Debug, Eq, PartialEq)]
pub struct ImporterRevisions {
    pub boot: String,
    pub schema: String,
    pub route: String,
    pub harness: String,
    pub check_policy: String,
}

#[derive(Clone, Debug, Eq, PartialEq)]
pub struct ImportedPackage {
    name: String,
    version: Version,
    source_sha256: [u8; 32],
}

impl ImportedPackage {
    pub fn name(&self) -> &str {
        &self.name
    }

    pub fn version(&self) -> &Version {
        &self.version
    }

    pub const fn source_sha256(&self) -> [u8; 32] {
        self.source_sha256
    }
}

#[derive(Clone)]
pub struct WebBootstrapImporter {
    inner: Arc<Inner>,
}

struct Inner {
    cache_root: PathBuf,
    projection_root: PathBuf,
    revisions: ImporterRevisions,
    podman: WebPodmanConfig,
    projection: Arc<K1WebProjection>,
}

impl WebBootstrapImporter {
    pub fn new(
        cache_root: impl Into<PathBuf>,
        projection_root: impl Into<PathBuf>,
        revisions: ImporterRevisions,
        podman: WebPodmanConfig,
        projection: Arc<K1WebProjection>,
    ) -> Result<Self, String> {
        let cache_root = cache_root.into();
        let projection_root = projection_root.into();
        require_directory(&cache_root, "Web bootstrap cache")?;
        require_directory(&projection_root, "Web projection")?;
        if [
            &revisions.boot,
            &revisions.schema,
            &revisions.route,
            &revisions.harness,
            &revisions.check_policy,
        ]
        .iter()
        .any(|value| value.is_empty())
        {
            return Err("Web bootstrap importer revisions must be nonempty".to_owned());
        }
        Ok(Self {
            inner: Arc::new(Inner {
                cache_root,
                projection_root,
                revisions,
                podman,
                projection,
            }),
        })
    }

    pub fn import_all(
        &self,
        archive: &Archive,
        authority: [u8; 12],
        user: [u8; 12],
    ) -> Result<Vec<ImportedPackage>, String> {
        let authority = AuthorityId::new(TxId::from_bytes(authority));
        let prepared = prepare_all(archive, authority)?;

        let revisions = &self.inner.revisions;
        let config = WebCodingConfig::new(
            WebCodingRevisions {
                boot: revisions.boot.clone(),
                schema: revisions.schema.clone(),
                route: revisions.route.clone(),
                harness: revisions.harness.clone(),
                check_policy: revisions.check_policy.clone(),
            },
            self.inner.projection_root.clone(),
            self.inner.podman.clone(),
        )
        .map_err(|error| format!("configure Web bootstrap coding: {error}"))?;
        let mut coding = K1WebCoding::open(
            &self.inner.cache_root,
            TxId::from_bytes(user),
            config,
            Arc::clone(&self.inner.projection),
        )
        .map_err(|error| format!("open Web bootstrap coding: {error}"))?;
        let authorize = |family: &WebFamily| Ok(family.authority() == authority);

        let mut imported = Vec::with_capacity(prepared.len());
        for item in prepared {
            let label = format!("{} {}", item.coordinate.name, item.coordinate.version);
            coding
                .write(&item.source, &authorize)
                .map_err(|error| format!("stage Web bootstrap package {label}: {error}"))?;
            let check = coding
                .check_fresh(item.source.id(), &authorize)
                .map_err(|error| format!("check Web bootstrap package {label}: {error}"))?;
            if !matches!(check, CheckOutcome::Checked(_)) {
                return Err(format!("fresh Web check was reused for {label}"));
            }

            match self
                .inner
                .projection
                .load(item.source.id())
                .map_err(|error| format!("inspect published Web package {label}: {error}"))?
            {
                Some(existing) if existing.as_ref() == &item.source => {}
                Some(_) => {
                    return Err(format!(
                        "published Web package {label} differs from bootstrap source"
                    ));
                }
                None => {
                    let completion = coding
                        .publish(item.source.id(), &authorize, &|_, _| Ok(true))
                        .map_err(|error| {
                            format!("publish Web bootstrap package {label}: {error}")
                        })?;
                    if completion.outcome.status == PublishStatus::Conflict {
                        return Err(format!(
                            "published Web package {label} conflicts with bootstrap source"
                        ));
                    }
                }
            }

            imported.push(ImportedPackage {
                name: item.coordinate.name,
                version: item.coordinate.version,
                source_sha256: item.source_sha256,
            });
        }
        Ok(imported)
    }
}

#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)]
struct Coordinate {
    name: String,
    version: Version,
}

struct PlannedPackage {
    index: usize,
    dependencies: Vec<GeneratedDependency>,
    edges: BTreeSet<Coordinate>,
}

#[derive(Debug)]
struct PreparedPackage {
    coordinate: Coordinate,
    source_sha256: [u8; 32],
    source: SourcePackage,
}

#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd, Serialize)]
struct GeneratedDependency {
    authority: String,
    name: String,
    selector: String,
}

#[derive(Serialize)]
struct Header<'a> {
    dependencies: &'a [GeneratedDependency],
}

#[derive(Serialize)]
struct Manifest<'a> {
    name: &'a str,
    version: String,
    entry: &'a str,
    tests: &'a str,
    dependencies: &'a [GeneratedDependency],
}

fn prepare_all(archive: &Archive, authority: AuthorityId) -> Result<Vec<PreparedPackage>, String> {
    let mut coordinates = BTreeMap::new();
    for (index, package) in archive.packages().iter().enumerate() {
        let coordinate = Coordinate {
            name: package.name().to_owned(),
            version: package.version().clone(),
        };
        if coordinates.insert(coordinate.clone(), index).is_some() {
            return Err(format!(
                "duplicate Web bootstrap package {} {}",
                coordinate.name, coordinate.version
            ));
        }
    }

    let authority_text = authority.to_string();
    let mut plans = BTreeMap::new();
    for (coordinate, index) in &coordinates {
        let package = &archive.packages()[*index];
        let mut dependencies = Vec::new();
        let mut edges = BTreeSet::new();

        for dependency in package.dependencies() {
            WebFamily::new(authority, dependency.name().to_owned()).map_err(|error| {
                format!(
                    "invalid dependency {} from {} {}: {error}",
                    dependency.name(),
                    coordinate.name,
                    coordinate.version
                )
            })?;
            let selector = DependencySelector::parse(dependency.selector()).map_err(|error| {
                format!(
                    "invalid selector {} for dependency {} from {} {}: {error}",
                    dependency.selector(),
                    dependency.name(),
                    coordinate.name,
                    coordinate.version
                )
            })?;

            if dependency.name() == coordinate.name && selector.matches(&coordinate.version) {
                continue;
            }

            let selected = coordinates
                .keys()
                .filter(|candidate| {
                    candidate.name == dependency.name() && selector.matches(&candidate.version)
                })
                .max_by(|left, right| left.version.cmp(&right.version))
                .cloned()
                .ok_or_else(|| {
                    format!(
                        "unresolved Web bootstrap dependency {} {} from {} {}",
                        dependency.name(),
                        dependency.selector(),
                        coordinate.name,
                        coordinate.version
                    )
                })?;

            edges.insert(selected);
            dependencies.push(GeneratedDependency {
                authority: authority_text.clone(),
                name: dependency.name().to_owned(),
                selector: dependency.selector().to_owned(),
            });
        }

        dependencies.sort();
        plans.insert(
            coordinate.clone(),
            PlannedPackage {
                index: *index,
                dependencies,
                edges,
            },
        );
    }

    let order = dependency_order(&plans)?;
    let mut prepared = Vec::with_capacity(order.len());
    for coordinate in order {
        let plan = plans
            .get(&coordinate)
            .expect("dependency order contains only planned coordinates");
        let package = &archive.packages()[plan.index];
        let source = prepare_package(package, authority, &plan.dependencies)?;
        prepared.push(PreparedPackage {
            coordinate,
            source_sha256: source_digest(&source),
            source,
        });
    }
    Ok(prepared)
}

fn dependency_order(
    plans: &BTreeMap<Coordinate, PlannedPackage>,
) -> Result<Vec<Coordinate>, String> {
    fn visit(
        coordinate: &Coordinate,
        plans: &BTreeMap<Coordinate, PlannedPackage>,
        states: &mut BTreeMap<Coordinate, u8>,
        stack: &mut Vec<Coordinate>,
        order: &mut Vec<Coordinate>,
    ) -> Result<(), String> {
        match states.get(coordinate).copied() {
            Some(2) => return Ok(()),
            Some(1) => {
                let start = stack
                    .iter()
                    .position(|item| item == coordinate)
                    .unwrap_or(0);
                let mut cycle = stack[start..].to_vec();
                cycle.push(coordinate.clone());
                let text = cycle
                    .iter()
                    .map(|item| format!("{} {}", item.name, item.version))
                    .collect::<Vec<_>>()
                    .join(" -> ");
                return Err(format!("non-self Web bootstrap dependency cycle: {text}"));
            }
            _ => {}
        }

        states.insert(coordinate.clone(), 1);
        stack.push(coordinate.clone());
        for dependency in &plans
            .get(coordinate)
            .expect("visited coordinate is planned")
            .edges
        {
            visit(dependency, plans, states, stack, order)?;
        }
        let popped = stack.pop();
        debug_assert_eq!(popped.as_ref(), Some(coordinate));
        states.insert(coordinate.clone(), 2);
        order.push(coordinate.clone());
        Ok(())
    }

    let mut states = BTreeMap::new();
    let mut stack = Vec::new();
    let mut order = Vec::with_capacity(plans.len());
    for coordinate in plans.keys() {
        visit(coordinate, plans, &mut states, &mut stack, &mut order)?;
    }
    Ok(order)
}

fn prepare_package(
    package: &Package,
    authority: AuthorityId,
    dependencies: &[GeneratedDependency],
) -> Result<SourcePackage, String> {
    let family = WebFamily::new(authority, package.name().to_owned())
        .map_err(|error| format!("assign Web package {} authority: {error}", package.name()))?;
    let id = WebId::new(family, package.version().clone())
        .map_err(|error| format!("assign Web package {} version: {error}", package.name()))?;
    let authority_text = authority.to_string();

    let original_documentation = package
        .files()
        .iter()
        .find(|file| file.path() == "Documentation.md")
        .ok_or_else(|| {
            format!(
                "Web bootstrap package {} {} lacks Documentation.md",
                package.name(),
                package.version()
            )
        })?
        .bytes();

    let header = serde_json::to_string(&Header { dependencies }).map_err(|error| {
        format!(
            "encode Web dependencies for {} {}: {error}",
            package.name(),
            package.version()
        )
    })?;
    let mut documentation = format!("<!-- k1-web/v1\n{header}\n-->\n").into_bytes();
    documentation.extend_from_slice(original_documentation);

    let manifest = serde_json::to_vec(&Manifest {
        name: package.name(),
        version: package.version().to_string(),
        entry: package.entry(),
        tests: package.tests(),
        dependencies,
    })
    .map_err(|error| {
        format!(
            "encode Web manifest for {} {}: {error}",
            package.name(),
            package.version()
        )
    })?;

    let mut files = Vec::with_capacity(package.files().len() + 1);
    for file in package.files() {
        if matches!(file.path(), "Documentation.md" | "k1-web.json") {
            continue;
        }
        files.push(SourceFile::new(
            file.path(),
            rewrite_file(
                file.path(),
                file.bytes(),
                &authority_text,
                package.dependencies(),
            ),
        ));
    }
    files.push(SourceFile::new("Documentation.md", documentation));
    files.push(SourceFile::new("k1-web.json", manifest));

    SourcePackage::new(id, files).map_err(|error| {
        format!(
            "construct Web bootstrap package {} {}: {error}",
            package.name(),
            package.version()
        )
    })
}

fn rewrite_file(
    path: &str,
    bytes: &[u8],
    authority: &str,
    dependencies: &[kcode_k1_web_bootstrap_archive::Dependency],
) -> Vec<u8> {
    if !matches!(extension(path), "js" | "mjs" | "html" | "css") {
        return bytes.to_vec();
    }
    let Ok(source) = std::str::from_utf8(bytes) else {
        return bytes.to_vec();
    };

    let mut rewritten = source.to_owned();
    let declarations = dependencies
        .iter()
        .map(|dependency| {
            (
                dependency.name().to_owned(),
                dependency.selector().to_owned(),
            )
        })
        .collect::<BTreeSet<_>>();
    for (name, selector) in declarations {
        let legacy = format!("/module/{name}/v{selector}/");
        let route_selector = if selector == "*" {
            "*".to_owned()
        } else {
            format!("v{selector}")
        };
        let current = format!("/{authority}/{name}/{route_selector}/");
        rewritten = rewritten.replace(&legacy, &current);
    }
    rewritten.into_bytes()
}

fn extension(path: &str) -> &str {
    path.rsplit_once('.').map_or("", |(_, extension)| extension)
}

fn source_digest(source: &SourcePackage) -> [u8; 32] {
    let mut hash = Sha256::new();
    for file in source.files() {
        hash.update((file.path().len() as u64).to_le_bytes());
        hash.update(file.path().as_bytes());
        hash.update((file.bytes().len() as u64).to_le_bytes());
        hash.update(file.bytes());
    }
    hash.finalize().into()
}

fn require_directory(path: &Path, label: &str) -> Result<(), String> {
    let metadata = std::fs::symlink_metadata(path)
        .map_err(|error| format!("inspect {label} {}: {error}", path.display()))?;
    if metadata.is_dir() && !metadata.file_type().is_symlink() {
        Ok(())
    } else {
        Err(format!("{label} is not an ordinary directory"))
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use kcode_k1_web_bootstrap_archive::read;
    use serde_json::Value;
    use std::fs::File;
    use std::io::Write;
    use zip::write::SimpleFileOptions;

    fn archive(root: &Path, name: &str, manifest: &str, entries: &[(&str, &[u8])]) -> Archive {
        let path = root.join(name);
        let file = File::create(&path).unwrap();
        let mut zip = zip::ZipWriter::new(file);
        zip.start_file("manifest.json", SimpleFileOptions::default())
            .unwrap();
        zip.write_all(manifest.as_bytes()).unwrap();
        for (path, bytes) in entries {
            zip.start_file(*path, SimpleFileOptions::default()).unwrap();
            zip.write_all(bytes).unwrap();
        }
        zip.finish().unwrap();
        read(&path).unwrap().archive
    }

    fn file<'a>(source: &'a SourcePackage, path: &str) -> &'a [u8] {
        source
            .files()
            .iter()
            .find(|file| file.path() == path)
            .unwrap()
            .bytes()
    }

    #[test]
    fn prepares_complete_trees_current_routes_and_exact_metadata() {
        let root = tempfile::tempdir().unwrap();
        let manifest = r#"{"schema":1,"packages":[{"name":"dep","version":"1.5.0","entry":"index.js","tests":"tests.js","dependencies":[]},{"name":"dep","version":"2.1.0","entry":"index.js","tests":"tests.js","dependencies":[]},{"name":"bridge","version":"1.0.0","entry":"index.js","tests":"tests.js","dependencies":[{"name":"dep","selector":"2"}]},{"name":"app","version":"1.0.0","entry":"src/index.js","tests":"tests.js","dependencies":[{"name":"app","selector":"1"},{"name":"dep","selector":"1"}]}]}"#;
        let entries: Vec<(&str, &[u8])> = vec![
            (
                "packages/dep/1.5.0/Documentation.md",
                b"dep one docs\r\n",
            ),
            (
                "packages/dep/1.5.0/index.js",
                b"export const version=1;\n",
            ),
            (
                "packages/dep/1.5.0/tests.js",
                b"export function runTests(){}\n",
            ),
            (
                "packages/dep/1.5.0/assets/deep/image.bin",
                &[0, 1, 255],
            ),
            (
                "packages/dep/2.1.0/Documentation.md",
                b"dep two docs\n",
            ),
            (
                "packages/dep/2.1.0/index.js",
                b"export const version=2;\n",
            ),
            (
                "packages/dep/2.1.0/tests.js",
                b"export function runTests(){}\n",
            ),
            (
                "packages/bridge/1.0.0/Documentation.md",
                b"bridge docs\n",
            ),
            (
                "packages/bridge/1.0.0/index.js",
                b"import '/module/dep/v2/index.js';\nexport {};\n",
            ),
            (
                "packages/bridge/1.0.0/tests.js",
                b"export function runTests(){}\n",
            ),
            (
                "packages/app/1.0.0/Documentation.md",
                b"app docs\r\n",
            ),
            (
                "packages/app/1.0.0/src/index.js",
                b"import '/module/dep/v1/index.js';\nexport const self='/module/app/v1/src/index.js';\n",
            ),
            (
                "packages/app/1.0.0/tests.js",
                b"export function runTests(){}\n",
            ),
            (
                "packages/app/1.0.0/styles/main.css",
                b"@import '/module/dep/v1/style.css';\n",
            ),
        ];
        let archive = archive(root.path(), "complete.zip", manifest, &entries);
        let authority = AuthorityId::new(TxId::from_bytes([1; 12]));
        let authority_text = authority.to_string();
        let prepared = prepare_all(&archive, authority).unwrap();

        let dep_one = prepared
            .iter()
            .find(|item| {
                item.coordinate.name == "dep" && item.coordinate.version == Version::new(1, 5, 0)
            })
            .unwrap();
        assert_eq!(file(&dep_one.source, "assets/deep/image.bin"), [0, 1, 255]);

        let app = prepared
            .iter()
            .find(|item| item.coordinate.name == "app")
            .unwrap();
        assert_eq!(app.source.entry(), "src/index.js");
        assert_eq!(app.source.tests(), "tests.js");
        assert_eq!(app.source.dependencies().len(), 1);
        assert_eq!(app.source.dependencies()[0].name(), "dep");
        let expected_documentation = format!(
            "<!-- k1-web/v1\n{{\"dependencies\":[{{\"authority\":\"{authority_text}\",\"name\":\"dep\",\"selector\":\"1\"}}]}}\n-->\napp docs\r\n"
        );
        assert_eq!(
            file(&app.source, "Documentation.md"),
            expected_documentation.as_bytes()
        );

        let generated: Value = serde_json::from_slice(file(&app.source, "k1-web.json")).unwrap();
        assert_eq!(generated["name"], "app");
        assert_eq!(generated["version"], "1.0.0");
        assert_eq!(generated["entry"], "src/index.js");
        assert_eq!(generated["tests"], "tests.js");
        assert_eq!(generated["dependencies"].as_array().unwrap().len(), 1);

        let code = std::str::from_utf8(file(&app.source, "src/index.js")).unwrap();
        assert!(code.contains(&format!("/{authority_text}/dep/v1/index.js")));
        assert!(code.contains(&format!("/{authority_text}/app/v1/src/index.js")));
        assert!(!code.contains("/module/"));
        assert!(!code.contains(&format!("/module/{authority_text}")));

        let style = std::str::from_utf8(file(&app.source, "styles/main.css")).unwrap();
        assert!(style.contains(&format!("/{authority_text}/dep/v1/style.css")));

        let dep_one_position = prepared
            .iter()
            .position(|item| {
                item.coordinate.name == "dep" && item.coordinate.version == Version::new(1, 5, 0)
            })
            .unwrap();
        let app_position = prepared
            .iter()
            .position(|item| item.coordinate.name == "app")
            .unwrap();
        let dep_two_position = prepared
            .iter()
            .position(|item| {
                item.coordinate.name == "dep" && item.coordinate.version == Version::new(2, 1, 0)
            })
            .unwrap();
        let bridge_position = prepared
            .iter()
            .position(|item| item.coordinate.name == "bridge")
            .unwrap();
        assert!(dep_one_position < app_position);
        assert!(dep_two_position < bridge_position);
    }

    #[test]
    fn rejects_unresolved_dependencies_and_non_self_cycles() {
        let root = tempfile::tempdir().unwrap();
        let unresolved_manifest = r#"{"schema":1,"packages":[{"name":"app","version":"1.0.0","entry":"index.js","tests":"tests.js","dependencies":[{"name":"missing","selector":"1"}]}]}"#;
        let app_entries: Vec<(&str, &[u8])> = vec![
            ("packages/app/1.0.0/Documentation.md", b"app docs\n"),
            ("packages/app/1.0.0/index.js", b"export {};\n"),
            (
                "packages/app/1.0.0/tests.js",
                b"export function runTests(){}\n",
            ),
        ];
        let unresolved = archive(
            root.path(),
            "unresolved.zip",
            unresolved_manifest,
            &app_entries,
        );
        let authority = AuthorityId::new(TxId::from_bytes([2; 12]));
        assert!(
            prepare_all(&unresolved, authority)
                .unwrap_err()
                .contains("unresolved Web bootstrap dependency")
        );

        let cycle_manifest = r#"{"schema":1,"packages":[{"name":"a","version":"1.0.0","entry":"index.js","tests":"tests.js","dependencies":[{"name":"b","selector":"1"}]},{"name":"b","version":"1.0.0","entry":"index.js","tests":"tests.js","dependencies":[{"name":"a","selector":"1"}]}]}"#;
        let cycle_entries: Vec<(&str, &[u8])> = vec![
            ("packages/a/1.0.0/Documentation.md", b"a docs\n"),
            ("packages/a/1.0.0/index.js", b"export {};\n"),
            (
                "packages/a/1.0.0/tests.js",
                b"export function runTests(){}\n",
            ),
            ("packages/b/1.0.0/Documentation.md", b"b docs\n"),
            ("packages/b/1.0.0/index.js", b"export {};\n"),
            (
                "packages/b/1.0.0/tests.js",
                b"export function runTests(){}\n",
            ),
        ];
        let cycle = archive(root.path(), "cycle.zip", cycle_manifest, &cycle_entries);
        assert!(
            prepare_all(&cycle, authority)
                .unwrap_err()
                .contains("non-self Web bootstrap dependency cycle")
        );
    }
}