diplodocus 0.15.0

Build coherent documentation websites for polyglot software projects.
Documentation
//! Run native Julia discovery with checkout and environment verification.

use std::collections::BTreeSet;
use std::process::Stdio;
use std::time::Duration;

use serde::{Deserialize, Serialize};

use super::*;
use crate::extractors::julia;
use crate::incremental::Manifest;

pub(super) struct JuliaNativeExtraction {
    pub(super) extraction: julia::JuliaExtraction,
    pub(super) completed: crate::ir::Provenance,
    pub(super) observations: Vec<Manifest>,
}

#[derive(Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
struct Cached {
    extraction: julia::JuliaExtraction,
    completed: crate::ir::Provenance,
    inputs: Manifest,
}

const STAGE: &str = "julia-native";

pub(super) fn portable_error(text: &str) -> String {
    text.lines()
        .map(|line| {
            // Julia's ProcessFailedException displays Cmd's whole environment.
            line.split_once("Process(setenv(").map_or_else(
                || line.to_owned(),
                |(prefix, rest)| {
                    let status = rest
                        .rsplit_once("ProcessExited(")
                        .and_then(|(_, suffix)| suffix.split_once(')'))
                        .map(|(code, _)| code)
                        .filter(|code| code.parse::<i32>().is_ok());
                    match status {
                        Some(code) => {
                            format!("{prefix}Julia subprocess exited with status {code}.")
                        }
                        None => format!("{prefix}Julia subprocess failed."),
                    }
                },
            )
        })
        .collect::<Vec<_>>()
        .join("\n")
}

pub(super) async fn results(
    path: &Path,
    deadlines: ExecutionDeadlines,
    cancellation: &mut ExecutionCancellation<'_>,
    reporting: &Reporting,
) -> Result<BTreeMap<(String, String), JuliaNativeExtraction>, AssemblyError> {
    let configuration_source =
        fs::read_to_string(path).map_err(|source| ConfigurationError::Read {
            path: path.to_owned(),
            source,
        })?;
    let configuration =
        parse_configuration(&configuration_source).map_err(|source| ConfigurationError::Parse {
            path: path.to_owned(),
            source,
        })?;
    reject_errors(validate_configuration_with_source(
        &configuration,
        "diplodocus.yaml".try_into().unwrap(),
    ))?;
    let paths = resolve_workspace_paths(path, &configuration)?;
    let config_input = Manifest::capture(&[std::path::absolute(path)?], &[])?;
    let mut progress = reporting.stage(
        "Extracting Julia APIs",
        Some(target_count(&configuration, "julia")),
    );
    let mut results = BTreeMap::new();
    for (package_index, (declaration, package)) in configuration
        .packages
        .iter()
        .zip(&paths.packages)
        .enumerate()
    {
        if !declaration
            .targets
            .iter()
            .any(|target| target.extractor == "julia")
        {
            continue;
        }
        let repository = &paths.repositories[package.repository_index];
        let mut project_paths: Vec<_> = [
            "Project.toml",
            "JuliaProject.toml",
            "Manifest.toml",
            "JuliaManifest.toml",
            "LocalPreferences.toml",
        ]
        .map(|filename| package.path.join(filename))
        .into();
        project_paths.extend((10..=13).flat_map(|minor| {
            [
                format!("Manifest-v1.{minor}.toml"),
                format!("JuliaManifest-v1.{minor}.toml"),
            ]
            .map(|filename| package.path.join(filename))
        }));
        project_paths.push(package.metadata_path.clone());
        let project_inputs = Manifest::capture(&project_paths, &[])?;
        let extraction_cache =
            crate::incremental::package_context(&configuration, &paths, package_index);
        let environment = if extraction_cache.is_some() {
            let validation = reporting.phase(Phase::CacheValidation, Some(&package.id), None);
            let environment = super::julia_environment::observe(
                &paths.configuration_directory,
                &package.path,
                &package.path,
                &package.path,
                deadlines,
                cancellation,
            )
            .await?;
            validation.finish();
            environment
        } else {
            None
        };
        for (target, selected) in declaration.targets.iter().zip(&package.targets) {
            if target.extractor != "julia" {
                continue;
            }
            progress.item(&format!("{}:{}", package.id, target.id));
            let cache = environment.as_ref().and_then(|environment| {
                super::julia_environment::Context::new(
                    extraction_cache.clone(),
                    environment,
                    STAGE,
                    &package.id,
                    &(declaration, target),
                    deadlines,
                )
            });
            let mut observations = cache
                .as_ref()
                .map_or_else(Vec::new, |cache| cache.manifests().to_vec());
            let cached = cache
                .as_ref()
                .and_then(|cache| cache.load::<Cached>(reporting))
                .filter(|candidate| {
                    candidate.extraction.target.package == package.id
                        && candidate.extraction.target.target == target.id
                        && candidate.inputs.verify()
                        && super::packages::evidence_current(
                            &candidate.extraction.provenance,
                            repository,
                        )
                });
            let cached = if let Some(cached) = cached {
                cached
            } else {
                if cache.is_none() && extraction_cache.is_some() {
                    reporting.cache(
                        STAGE,
                        &package.id,
                        false,
                        Some("Prepared Julia dependency coverage is uncertain."),
                    );
                }
                let failure = |message: String| {
                    AssemblyError::Diagnostics(vec![
                        Diagnostic::new(
                            DiagnosticCode::JuliaConflictingSurface,
                            Severity::Error,
                            message,
                        )
                        .with_entity(
                            crate::diagnostics::DiagnosticEntity::Target {
                                package: package.id.clone(),
                                id: target.id.clone(),
                            },
                        ),
                    ])
                };
                let before = Manifest::capture(std::slice::from_ref(&selected.path), &[])?;
                let staging = tempfile::tempdir()?;
                let output = staging.path().join("observations.toml");
                let errors = staging.path().join("errors");
                let helper = staging.path().join("runtime.jl");
                fs::write(
                    &helper,
                    super::julia_environment::completed_source(
                        include_str!("../extractors/julia/runtime.jl"),
                        staging.path(),
                    )?,
                )?;
                let mut command = tokio::process::Command::new("julia");
                command
                    .args(["--startup-file=no", "--history-file=no", "--color=no"])
                    .arg(format!("--project={}", package.path.display()))
                    .arg(&helper)
                    .arg(&package.path)
                    .arg(&selected.path)
                    .arg(&output)
                    .current_dir(&package.path)
                    .stdin(Stdio::null())
                    .stdout(Stdio::null())
                    .stderr(fs::File::create(&errors)?)
                    .kill_on_drop(true);
                #[cfg(target_os = "linux")]
                command.process_group(0);
                let timing =
                    reporting.phase(Phase::NativeProcess, Some(&package.id), Some(&target.id));
                let child = command.spawn().map_err(|error| failure(format!(
                    "Julia extraction requires Julia and the prepared checkout dependencies: {error}"
                )))?;
                #[cfg(target_os = "linux")]
                let group = child
                    .id()
                    .and_then(|pid| rustix::process::Pid::from_raw(pid as i32));
                let mut process = rd_examples::Process {
                    child,
                    #[cfg(target_os = "linux")]
                    group,
                };
                let status = tokio::select! {
                    result = tokio::time::timeout(Duration::from_millis(deadlines.cell.max(1)), process.child.wait()) => {
                        match result {
                            Ok(status) => status?,
                            Err(_) => {
                                rd_examples::terminate(&mut process, deadlines).await?;
                                return Err(failure("Julia native extraction timed out.".into()));
                            }
                        }
                    }
                    () = cancellation.as_mut() => {
                        rd_examples::terminate(&mut process, deadlines).await?;
                        return Err(AssemblyError::Cancelled);
                    }
                };
                drop(process);
                reporting.native_timings(
                    &output.with_extension("timings"),
                    &package.id,
                    Some(&target.id),
                );
                let portable = |text: &str| {
                    text.replace(repository.path.to_string_lossy().as_ref(), "<repository>")
                        .replace(staging.path().to_string_lossy().as_ref(), "<julia-helper>")
                };
                if !status.success() {
                    return Err(failure(format!(
                        "Julia runtime extraction failed: {}",
                        portable(&portable_error(&fs::read_to_string(errors)?)).trim()
                    )));
                }
                timing.finish();
                let observation: julia::runtime::Observation =
                    toml::from_str(&fs::read_to_string(output)?).map_err(|error| {
                        failure(format!(
                            "Invalid Julia runtime observations: {}",
                            portable(&error.to_string())
                        ))
                    })?;
                let observed_paths = observation.input_paths(package);
                let inputs = Manifest::capture(&observed_paths, &[])?;
                if !before.verify()
                    || !project_inputs.verify()
                    || !observation.verify(package).unwrap_or(false)
                {
                    return Err(AssemblyError::InputsChanged);
                }
                let cache_complete = observation.complete
                    && cache.as_ref().is_some_and(|cache| {
                        let covered: BTreeSet<_> =
                            cache.manifests().iter().flat_map(Manifest::paths).collect();
                        observed_paths.iter().all(|path| covered.contains(path))
                            && cache.accepts_completion(staging.path(), reporting)
                    });
                let timing =
                    reporting.phase(Phase::Extraction, Some(&package.id), Some(&target.id));
                let (mut extraction, completed) =
                    julia::runtime::reconcile(repository, package, selected, observation)
                        .map_err(|message| failure(portable(&message)))?;
                for diagnostic in &mut extraction.diagnostics {
                    diagnostic.message = portable(&portable_error(&diagnostic.message));
                    diagnostic.related_entity.get_or_insert_with(|| {
                        crate::diagnostics::DiagnosticEntity::Target {
                            package: package.id.clone(),
                            id: target.id.clone(),
                        }
                    });
                }
                reject_errors(extraction.diagnostics.clone())?;
                if !inputs.verify()
                    || !before.verify()
                    || !project_inputs.verify()
                    || !super::packages::evidence_current(&extraction.provenance, repository)
                {
                    return Err(AssemblyError::InputsChanged);
                }
                timing.finish();
                staging.close()?;
                let value = Cached {
                    extraction,
                    completed,
                    inputs,
                };
                if let Some(cache) = &cache
                    && cache_complete
                {
                    cache.save(&value);
                }
                value
            };
            observations.extend([cached.inputs, project_inputs.clone(), config_input.clone()]);
            results.insert(
                (package.id.clone(), target.id.clone()),
                JuliaNativeExtraction {
                    extraction: cached.extraction,
                    completed: cached.completed,
                    observations,
                },
            );
            progress.advance();
        }
    }
    if configuration_source.as_bytes() != fs::read(path)? {
        return Err(AssemblyError::InputsChanged);
    }
    progress.finish();
    Ok(results)
}

pub(crate) async fn assemble_for_extraction(
    path: &Path,
    deadlines: ExecutionDeadlines,
    cancellation: &mut ExecutionCancellation<'_>,
    reporting: &Reporting,
    journal: &mut super::r_native::GeneratedWrites,
    bundle_export: bool,
) -> Result<WorkspaceSources, AssemblyError> {
    if bundle_export {
        super::bundles::validate_export(&crate::configuration::load_configuration(path)?)?;
    }
    let julia = results(path, deadlines, cancellation, reporting).await?;
    let rust = super::rust_results(path, true, deadlines, cancellation, reporting).await?;
    let doxygen = super::doxygen_results(path, deadlines, cancellation, reporting).await?;
    let r = super::r_results(path, deadlines, cancellation, reporting, journal).await?;
    let python = super::python_native::results(path, deadlines, cancellation, reporting).await?;
    super::assemble_with_native(
        path,
        super::NativeExtractions {
            julia,
            rust,
            doxygen,
            r,
            python,
        },
        bundle_export,
        reporting,
        cancellation,
    )
    .await
}