use std::collections::BTreeMap;
use std::fs;
use std::path::{Path, PathBuf};
use crate::configuration::{ConfigurationError, WorkspaceConfiguration, parse_configuration};
use crate::configuration_validation::validate_configuration_with_source;
use crate::diagnostics::{Diagnostic, DiagnosticCode, DiagnosticPath, DiagnosticSource, Severity};
use crate::documents::{QmdPreparation, includes};
use crate::execution::{
ExecutionCancellation, ExecutionDeadlines, ExecutionFailure, PageExecutionResult,
};
use crate::ir::{Fingerprint, SourceLocation, Workspace};
use crate::paths::{PathResolutionError, ResolvedWorkspacePaths, resolve_workspace_paths};
use crate::provenance::{
DeclaredSourceInputs, ProvenanceError, StaticProvenance, collect_static_provenance,
fingerprint_bytes,
};
use crate::reporting::{Phase, Reporting};
mod adapters;
mod api_examples;
mod authored;
mod bundles;
mod documenter;
mod documenter_examples;
mod doxygen_native;
mod julia_environment;
mod julia_runtime;
pub(crate) use julia_runtime::assemble_for_extraction;
#[cfg(target_os = "linux")]
mod execution;
mod package_docs;
mod packages;
mod python_native;
pub(crate) mod r_native;
mod rd_examples;
mod references;
mod reuse;
mod rmarkdown;
mod rust_native;
pub use crate::ir::IncludeSourceRange;
pub(crate) use includes::remap as remap_include_diagnostic;
pub(crate) use includes::watch_inputs as include_watch_inputs;
pub(crate) const PACKAGE_LICENSE_FILENAMES: &[&str] = &[
"LICENSE",
"LICENSE.md",
"LICENCE",
"LICENCE.md",
"LICENSE-MIT",
"LICENSE-APACHE",
"COPYING",
"COPYING.md",
];
#[derive(Debug, thiserror::Error)]
pub enum AssemblyError {
#[error("package bundle {path}: {message}")]
Bundle {
path: PathBuf,
message: String,
},
#[error(transparent)]
Configuration(#[from] ConfigurationError),
#[error(transparent)]
Path(#[from] PathResolutionError),
#[error(transparent)]
Provenance(#[from] ProvenanceError),
#[error("workspace assembly failed with source diagnostics")]
Diagnostics(Vec<Diagnostic>),
#[error("workspace source or staging I/O failed: {0}")]
Io(#[from] std::io::Error),
#[error("workspace inputs changed during assembly or execution")]
InputsChanged,
#[error("workspace execution was canceled")]
Cancelled,
#[error(transparent)]
Execution(#[from] ExecutionFailure),
#[error("{cause}; workspace staging cleanup also failed: {source}")]
Cleanup {
cause: Box<AssemblyError>,
source: std::io::Error,
},
}
impl From<includes::IncludeError> for AssemblyError {
fn from(error: includes::IncludeError) -> Self {
match error {
includes::IncludeError::Path(error) => Self::Path(error),
includes::IncludeError::Diagnostics(diagnostics) => Self::Diagnostics(diagnostics),
}
}
}
#[derive(Debug)]
pub struct PreparedPage {
pub(crate) source_ranges: Vec<IncludeSourceRange>,
pub(crate) included_resource_targets: std::collections::BTreeSet<String>,
pub(crate) source: String,
pub(crate) path: PathBuf,
pub(crate) relative: DiagnosticPath,
pub(crate) location: SourceLocation,
pub(crate) collection: usize,
pub(crate) preparation: Option<QmdPreparation>,
}
impl PreparedPage {
pub fn source_ranges(&self) -> &[IncludeSourceRange] {
&self.source_ranges
}
pub fn source(&self) -> &str {
&self.source
}
pub fn preparation(&self) -> Option<&QmdPreparation> {
self.preparation.as_ref()
}
}
#[derive(Debug)]
pub struct WorkspaceSources {
reporting: Reporting,
workspace: Workspace,
configuration: WorkspaceConfiguration,
configuration_path: PathBuf,
configuration_source: String,
paths: ResolvedWorkspacePaths,
prepared_pages: BTreeMap<String, PreparedPage>,
package_files: BTreeMap<String, package_docs::PackageFiles>,
rmd_inputs: BTreeMap<String, Vec<PathBuf>>,
doxygen_observations: Vec<doxygen_native::DoxygenObservation>,
r_observations: Vec<r_native::RObservation>,
cache_observations: CacheObservations,
python_interpreter: Option<PathBuf>,
python_doctests: BTreeMap<String, crate::extractors::python::native::DoctestPlans>,
python_environments: BTreeMap<
String,
(
api_examples::PythonEnvironment,
crate::incremental::Manifest,
),
>,
selections: DeclaredSourceInputs,
evidence: StaticProvenance,
pub(crate) bundles: bundles::ImportedBundles,
bundle_export: bool,
pub(crate) native_assets: BTreeMap<DiagnosticPath, crate::validation::ContentAsset>,
}
#[derive(Debug, Default)]
struct CacheObservations(Vec<crate::incremental::Manifest>);
impl CacheObservations {
fn push(&mut self, manifest: crate::incremental::Manifest) {
if !self.contains(&manifest) {
self.0.push(manifest);
}
}
fn contains(&self, manifest: &crate::incremental::Manifest) -> bool {
self.0.contains(manifest)
}
fn iter(&self) -> impl Iterator<Item = &crate::incremental::Manifest> {
self.0.iter()
}
}
impl Extend<crate::incremental::Manifest> for CacheObservations {
fn extend<T: IntoIterator<Item = crate::incremental::Manifest>>(&mut self, manifests: T) {
for manifest in manifests {
self.push(manifest);
}
}
}
impl WorkspaceSources {
pub fn workspace(&self) -> &Workspace {
&self.workspace
}
pub fn prepared_pages(&self) -> &BTreeMap<String, PreparedPage> {
&self.prepared_pages
}
pub fn configuration(&self) -> &WorkspaceConfiguration {
&self.configuration
}
pub fn paths(&self) -> &ResolvedWorkspacePaths {
&self.paths
}
pub fn input_paths(&self) -> impl Iterator<Item = PathBuf> + '_ {
self.evidence
.inputs
.iter()
.flat_map(|(repository, files)| {
let root = &self
.paths
.repositories
.iter()
.find(|r| &r.id == repository)
.expect("observed repository")
.path;
files.keys().map(move |path| root.join(path.as_str()))
})
.chain(self.bundles.input_paths())
.chain(
self.r_observations
.iter()
.flat_map(|observation| observation.paths().cloned()),
)
.chain(
self.cache_observations
.iter()
.flat_map(|manifest| manifest.paths().cloned()),
)
}
pub(crate) fn doxygen_watch_roots(&self) -> impl Iterator<Item = &PathBuf> {
self.doxygen_observations
.iter()
.flat_map(|observation| observation.roots())
}
pub(crate) fn dependency_watch_roots(&self) -> impl Iterator<Item = &PathBuf> {
self.cache_observations
.iter()
.flat_map(|manifest| manifest.roots())
.chain(
self.r_observations
.iter()
.flat_map(|observation| observation.dependency_roots()),
)
}
pub(crate) fn dependency_library_roots(&self) -> impl Iterator<Item = &PathBuf> {
self.cache_observations
.iter()
.flat_map(|manifest| manifest.native_library_roots())
.chain(
self.r_observations
.iter()
.flat_map(|observation| observation.dependency_library_roots()),
)
}
pub fn revalidate(&self) -> Result<(), AssemblyError> {
let timing = self.reporting.phase(Phase::SourceVerification, None, None);
let result = crate::incremental::verification_scope(|| self.revalidate_inner());
if result.is_ok() {
timing.finish();
}
result
}
fn revalidate_inner(&self) -> Result<(), AssemblyError> {
if self
.cache_observations
.iter()
.any(|manifest| !manifest.verify())
{
return Err(AssemblyError::InputsChanged);
}
for observation in &self.r_observations {
observation.verify()?;
}
for observation in &self.doxygen_observations {
observation.verify()?;
}
if fs::read(&self.configuration_path)? != self.configuration_source.as_bytes() {
return Err(AssemblyError::InputsChanged);
}
let mut configuration =
parse_configuration(&self.configuration_source).map_err(|source| {
ConfigurationError::Parse {
path: self.configuration_path.clone(),
source,
}
})?;
self.bundles.revalidate()?;
if !self.bundle_export {
rmarkdown::collections(&self.configuration_path, &mut configuration)?;
}
if configuration != self.configuration {
return Err(AssemblyError::InputsChanged);
}
let paths = resolve_workspace_paths(&self.configuration_path, &self.configuration)?;
if paths != self.paths {
return Err(AssemblyError::InputsChanged);
}
for (index, collection) in self.configuration.content.iter().enumerate() {
let content = &paths.content[index];
if authored::discover(
&content.path,
collection.format,
&paths.repositories[content.repository_index],
)? != self.selections.content[&collection.id]
{
return Err(AssemblyError::InputsChanged);
}
}
if package_docs::discover(&paths, &self.workspace)? != self.package_files {
return Err(AssemblyError::InputsChanged);
}
if rmarkdown::inputs(&paths, &self.configuration)? != self.rmd_inputs {
return Err(AssemblyError::InputsChanged);
}
let current = collect_static_provenance(
&self.configuration_path,
&self.configuration,
&self.selections,
)?;
if current.inputs != self.evidence.inputs {
return Err(AssemblyError::InputsChanged);
}
Ok(())
}
}
pub struct WorkspaceExecution<'a> {
pub staging_parent: &'a Path,
pub deadlines: ExecutionDeadlines,
pub cancellation: ExecutionCancellation<'a>,
}
#[derive(Debug)]
pub struct ExecutedWorkspace {
sources: WorkspaceSources,
pages: BTreeMap<String, PageExecutionResult>,
staging: Option<tempfile::TempDir>,
}
impl ExecutedWorkspace {
pub fn workspace(&self) -> &Workspace {
self.sources.workspace()
}
pub fn executed_pages(&self) -> &BTreeMap<String, PageExecutionResult> {
&self.pages
}
pub fn sources(&self) -> &WorkspaceSources {
&self.sources
}
pub fn discard(mut self) -> Result<(), AssemblyError> {
self.pages.clear();
if let Some(staging) = self.staging.take() {
staging.close()?;
}
Ok(())
}
}
pub fn assemble_workspace(path: impl AsRef<Path>) -> Result<WorkspaceSources, AssemblyError> {
let path = path.as_ref().to_owned();
std::thread::Builder::new()
.name("native-documentation-check".into())
.spawn(move || {
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()?;
runtime.block_on(async {
let mut cancellation: ExecutionCancellation<'_> = Box::pin(std::future::pending());
assemble_for_check(
&path,
ExecutionDeadlines::default(),
&mut cancellation,
&Reporting::default(),
)
.await
})
})?
.join()
.map_err(|_| {
AssemblyError::Io(std::io::Error::other(
"Native documentation check thread failed",
))
})?
}
async fn rust_results(
path: &Path,
run_examples: bool,
deadlines: ExecutionDeadlines,
cancellation: &mut ExecutionCancellation<'_>,
reporting: &Reporting,
) -> Result<BTreeMap<(String, String), rust_native::RustNativeExtraction>, AssemblyError> {
let loading = reporting.stage("Loading workspace", None);
let configuration_source = fs::read(path)?;
let configuration = crate::configuration::load_configuration(path)?;
reject_errors(validate_configuration_with_source(
&configuration,
"diplodocus.yaml".try_into().unwrap(),
))?;
let paths = resolve_workspace_paths(path, &configuration)?;
loading.finish();
let mut progress = reporting.stage(
"Extracting Rust APIs",
Some(target_count(&configuration, "rust")),
);
let cache = paths.configuration_directory.join(".diplodocus/cache");
let mut results = BTreeMap::new();
for (index, (declaration, package)) in configuration
.packages
.iter()
.zip(&paths.packages)
.enumerate()
{
let repository = &paths.repositories[package.repository_index];
let result_cache = crate::incremental::package_context(&configuration, &paths, index);
for (target, selected) in declaration.targets.iter().zip(&package.targets) {
if target.extractor == "rust" {
progress.item(&format!("{}:{}", package.id, target.id));
let result = rust_native::extract_target_cached(
repository,
package,
selected,
&target.rust.clone().unwrap_or_default(),
&cache,
run_examples,
reporting,
deadlines,
cancellation,
result_cache.as_ref(),
)
.await?;
results.insert((package.id.clone(), target.id.clone()), result);
progress.advance();
}
}
}
if fs::read(path)? != configuration_source {
return Err(AssemblyError::InputsChanged);
}
progress.finish();
Ok(results)
}
pub(crate) async fn assemble_for_check(
path: &Path,
deadlines: ExecutionDeadlines,
cancellation: &mut ExecutionCancellation<'_>,
reporting: &Reporting,
) -> Result<WorkspaceSources, AssemblyError> {
let rust = rust_results(path, false, deadlines, cancellation, reporting).await?;
let doxygen = doxygen_results(path, deadlines, cancellation, reporting).await?;
let r = r_results(
path,
deadlines,
cancellation,
reporting,
&mut r_native::GeneratedWrites::default(),
)
.await?;
let python = python_native::results(path, deadlines, cancellation, reporting).await?;
let julia = julia_runtime::results(path, deadlines, cancellation, reporting).await?;
assemble_with_native(
path,
NativeExtractions {
julia,
rust,
doxygen,
r,
python,
},
false,
reporting,
cancellation,
)
.await
}
async fn doxygen_results(
path: &Path,
deadlines: ExecutionDeadlines,
cancellation: &mut ExecutionCancellation<'_>,
reporting: &Reporting,
) -> Result<BTreeMap<(String, String), doxygen_native::DoxygenNativeExtraction>, AssemblyError> {
let before = fs::read(path)?;
let configuration = crate::configuration::load_configuration(path)?;
reject_errors(validate_configuration_with_source(
&configuration,
"diplodocus.yaml".try_into().unwrap(),
))?;
let paths = resolve_workspace_paths(path, &configuration)?;
let mut progress = reporting.stage(
"Extracting C++ APIs",
Some(target_count(&configuration, "doxygen")),
);
let mut results = BTreeMap::new();
for (index, (declaration, package)) in configuration
.packages
.iter()
.zip(&paths.packages)
.enumerate()
{
let repository = &paths.repositories[package.repository_index];
let cache = crate::incremental::package_context(&configuration, &paths, index);
for (target, selected) in declaration.targets.iter().zip(&package.targets) {
if target.extractor == "doxygen" {
progress.item(&format!("{}:{}", package.id, target.id));
results.insert(
(package.id.clone(), target.id.clone()),
doxygen_native::extract_target(
repository,
package,
selected,
reporting,
deadlines,
cancellation,
cache.as_ref(),
)
.await?,
);
progress.advance();
}
}
}
if fs::read(path)? != before {
return Err(AssemblyError::InputsChanged);
}
progress.finish();
Ok(results)
}
async fn r_results(
path: &Path,
deadlines: ExecutionDeadlines,
cancellation: &mut ExecutionCancellation<'_>,
reporting: &Reporting,
journal: &mut r_native::GeneratedWrites,
) -> Result<BTreeMap<(String, String), r_native::RNativeExtraction>, AssemblyError> {
let before = fs::read(path)?;
let configuration = crate::configuration::load_configuration(path)?;
reject_errors(validate_configuration_with_source(
&configuration,
"diplodocus.yaml".try_into().unwrap(),
))?;
let paths = resolve_workspace_paths(path, &configuration)?;
let configuration_path = fs::canonicalize(path)?;
for (declaration, package) in configuration.packages.iter().zip(&paths.packages) {
if let Some(settings) = &declaration.r {
for output in &settings.generated_outputs {
let output = r_native::resolved_output(package, output)?;
if configuration_path.starts_with(&output)
|| output.starts_with(&configuration_path)
|| paths.packages.iter().zip(&configuration.packages).any(
|(other, declaration)| {
other.id != package.id
&& (other.path.starts_with(&output)
|| (!package.path.starts_with(&other.path)
&& output.starts_with(&other.path))
|| other.metadata_path.starts_with(&output)
|| output.starts_with(&other.metadata_path)
|| declaration.r.as_ref().is_some_and(|settings| {
settings
.generated_outputs
.iter()
.chain(&settings.extra_inputs)
.any(|path| {
let path = other.path.join(path);
path.starts_with(&output)
|| output.starts_with(&path)
})
}))
},
)
{
return Err(AssemblyError::Diagnostics(vec![Diagnostic::new(
DiagnosticCode::InvalidConfiguration,
Severity::Error,
"Generated R outputs must not overlap workspace configuration or another package.",
)]));
}
}
}
}
let results = r_native::results(
&configuration,
&paths,
deadlines,
cancellation,
reporting,
journal,
)
.await?;
if fs::read(path)? != before {
return Err(AssemblyError::InputsChanged);
}
Ok(results)
}
struct NativeExtractions {
python: BTreeMap<(String, String), python_native::PythonNativeExtraction>,
julia: BTreeMap<(String, String), julia_runtime::JuliaNativeExtraction>,
rust: BTreeMap<(String, String), rust_native::RustNativeExtraction>,
doxygen: BTreeMap<(String, String), doxygen_native::DoxygenNativeExtraction>,
r: BTreeMap<(String, String), r_native::RNativeExtraction>,
}
async fn assemble_with_native(
path: &Path,
mut native: NativeExtractions,
bundle_export: bool,
reporting: &Reporting,
cancellation: &mut ExecutionCancellation<'_>,
) -> Result<WorkspaceSources, AssemblyError> {
let text = fs::read_to_string(path).map_err(|source| ConfigurationError::Read {
path: path.to_owned(),
source,
})?;
let mut configuration =
parse_configuration(&text).map_err(|source| ConfigurationError::Parse {
path: path.to_owned(),
source,
})?;
let config_name = DiagnosticPath::try_from(
path.file_name()
.and_then(|s| s.to_str())
.unwrap_or("diplodocus.yaml"),
)
.map_err(|_| AssemblyError::InputsChanged)?;
reject_errors(validate_configuration_with_source(
&configuration,
config_name.clone(),
))?;
if bundle_export {
bundles::validate_export(&configuration)?;
} else {
rmarkdown::collections(path, &mut configuration)?;
}
reject_errors(validate_configuration_with_source(
&configuration,
config_name.clone(),
))?;
let paths = resolve_workspace_paths(path, &configuration)?;
let mut workspace = Workspace {
name: configuration.project.name.clone(),
..Workspace::default()
};
let mut selections = DeclaredSourceInputs::default();
let mut observed = BTreeMap::new();
let mut cache_observations = CacheObservations::default();
cache_observations.extend(
native
.rust
.values()
.filter_map(|result| result.observation.clone()),
);
cache_observations.extend(
native
.python
.values()
.flat_map(|result| result.observations.iter().cloned()),
);
cache_observations.extend(
native
.julia
.values()
.flat_map(|result| result.observations.iter().cloned()),
);
for package in &configuration.packages {
cache_observations.push(crate::incremental::supplemental_manifest(
&configuration,
&paths,
package,
)?);
}
for (index, collection) in configuration.content.iter().enumerate() {
let repository = &paths.repositories[paths.content[index].repository_index];
let inputs = collection
.incremental
.extra_inputs
.iter()
.map(|path| crate::incremental::checked_incremental_path(&repository.path, path))
.collect::<Result<Vec<_>, _>>()?;
cache_observations.push(crate::incremental::Manifest::capture(
&inputs,
&collection.incremental.environment_variables,
)?);
}
let doxygen_observations = native
.doxygen
.values()
.map(|result| result.observation.clone())
.collect();
let r_observations = native
.r
.values()
.map(|result| result.observation.clone())
.collect();
let python_interpreter = native
.python
.values()
.next()
.map(|result| result.interpreter.clone());
let python_environments = native
.python
.iter()
.filter_map(|((package, _), result)| {
let environment = result.environment.as_ref()?;
native
.python
.iter()
.filter(|((owner, _), _)| owner == package)
.all(|(_, result)| {
result
.environment
.as_ref()
.is_some_and(|(candidate, _)| candidate == &environment.0)
})
.then(|| (package.clone(), environment.clone()))
})
.collect();
let rmd_inputs = rmarkdown::inputs(&paths, &configuration)?;
for (collection, resolved) in configuration.content.iter().zip(&paths.content) {
if let Some(files) = rmd_inputs.get(&collection.id) {
let repository = &paths.repositories[resolved.repository_index];
selections
.package_documentation
.entry(repository.id.clone())
.or_default()
.extend(files.iter().cloned());
for file in files {
let path = repository.path.join(file);
observe(
&mut observed,
&repository.source_location(&path, None)?,
fingerprint_bytes(&fs::read(path)?),
)?;
}
}
}
let python_doctests = packages::assemble(
&configuration,
&paths,
&mut workspace,
&mut selections,
&mut observed,
&mut native,
reporting,
)?;
let prepared_pages = authored::assemble(
&configuration,
&paths,
&mut workspace,
&mut selections,
&mut observed,
reporting,
)?;
let package_files = package_docs::assemble(
&paths,
&mut workspace,
&prepared_pages,
&mut selections,
&mut observed,
)?;
let evidence = collect_static_provenance(path, &configuration, &selections)?;
for ((repository, path), fingerprint) in observed {
if evidence
.inputs
.get(&repository)
.and_then(|inputs| inputs.get(&path))
!= Some(&fingerprint)
{
return Err(AssemblyError::InputsChanged);
}
}
workspace.repositories = evidence.repositories.clone();
let bundles = bundles::ImportedBundles::load(
&configuration,
&paths.configuration_directory,
&mut workspace,
cancellation,
)
.await?;
documenter::assemble(&mut workspace, &prepared_pages);
references::concepts_and_relationships(&configuration, &mut workspace, &config_name);
reject_errors(workspace.diagnostics.iter().cloned().collect())?;
let result = WorkspaceSources {
reporting: reporting.clone(),
workspace,
configuration,
configuration_path: std::env::current_dir()?.join(path),
configuration_source: text,
paths,
prepared_pages,
package_files,
rmd_inputs,
doxygen_observations,
r_observations,
cache_observations,
python_interpreter,
python_doctests,
python_environments,
selections,
evidence,
bundles,
bundle_export,
native_assets: BTreeMap::new(),
};
result.revalidate()?;
Ok(result)
}
fn target_count(configuration: &WorkspaceConfiguration, extractor: &str) -> usize {
configuration
.packages
.iter()
.flat_map(|package| &package.targets)
.filter(|target| target.extractor == extractor)
.count()
}
type ObservedInputs = BTreeMap<(String, DiagnosticPath), Fingerprint>;
fn observe(
inputs: &mut ObservedInputs,
source: &SourceLocation,
fingerprint: Fingerprint,
) -> Result<(), AssemblyError> {
if inputs
.insert(
(source.repository.clone(), source.path.clone()),
fingerprint.clone(),
)
.is_some_and(|old| old != fingerprint)
{
return Err(AssemblyError::InputsChanged);
}
Ok(())
}
fn reject_errors(diagnostics: Vec<Diagnostic>) -> Result<(), AssemblyError> {
if diagnostics.iter().any(|d| d.severity == Severity::Error) {
Err(AssemblyError::Diagnostics(diagnostics))
} else {
Ok(())
}
}
fn diagnostic(code: DiagnosticCode, message: impl Into<String>) -> Diagnostic {
Diagnostic::new(code, Severity::Error, message)
}
fn portable(path: &Path) -> Result<DiagnosticPath, AssemblyError> {
DiagnosticPath::try_from(path.to_str().ok_or(AssemblyError::InputsChanged)?)
.map_err(|_| AssemblyError::InputsChanged)
}
fn relative(path: &Path, base: &Path) -> Result<Option<DiagnosticPath>, AssemblyError> {
let relative = path
.strip_prefix(base)
.map_err(|_| AssemblyError::InputsChanged)?;
if relative.as_os_str().is_empty() {
Ok(None)
} else {
portable(relative).map(Some)
}
}
fn source(source: &SourceLocation) -> DiagnosticSource {
DiagnosticSource::Repository {
repository: source.repository.clone(),
path: source.path.clone(),
}
}