use std::collections::{BTreeMap, BTreeSet};
use std::fs;
use std::path::{Path, PathBuf};
use serde::{Deserialize, Serialize};
use crate::diagnostics::{Diagnostic, DiagnosticCode, DiagnosticPath, DiagnosticSource, Severity};
use crate::ir::*;
use crate::paths::{
ResolvedPackagePaths, ResolvedRepositoryPaths, ResolvedTargetPath, resolve_input_file,
};
use crate::provenance::{BuiltinExtractor, ExtractionObservation, fingerprint_bytes};
pub mod docstrings;
mod metadata;
pub(crate) mod runtime;
mod signatures;
mod source;
mod surface;
pub use metadata::{JuliaMetadata, JuliaMetadataField};
pub(crate) const FATOU_VERSION: &str = env!("DIPLODOCUS_FATOU_VERSION");
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct JuliaExtraction {
pub schema_version: SchemaVersion,
pub package: String,
pub target: TargetReference,
pub metadata: Option<JuliaMetadata>,
pub items: BTreeMap<String, Item>,
pub inputs: BTreeMap<DiagnosticPath, JuliaInput>,
pub diagnostics: Vec<Diagnostic>,
pub provenance: Provenance,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct JuliaInput {
pub kind: String,
pub source: SourceLocation,
pub text: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub raw_bytes: Option<Vec<u8>>,
}
fn valid_target(
repository: &ResolvedRepositoryPaths,
package: &ResolvedPackagePaths,
target: &ResolvedTargetPath,
) -> bool {
package.path.starts_with(&repository.path)
&& fs::canonicalize(&package.path).ok().as_ref() == Some(&package.path)
&& target.path.starts_with(&package.path)
&& target.path.is_file()
&& target
.path
.extension()
.is_some_and(|extension| extension == "jl")
}
fn prepare_extraction(
repository: &ResolvedRepositoryPaths,
package: &ResolvedPackagePaths,
target: &ResolvedTargetPath,
capabilities: &[&str],
) -> (JuliaExtraction, ExtractionObservation) {
let reference = TargetReference {
package: package.id.clone(),
target: target.id.clone(),
};
let mut observation = ExtractionObservation {
target: reference.clone(),
extractor: BuiltinExtractor::Julia,
capabilities: capabilities.iter().map(|s| (*s).into()).collect(),
parsers: BTreeMap::new(),
inputs: BTreeMap::new(),
};
let mut result = JuliaExtraction {
schema_version: SchemaVersion,
package: package.id.clone(),
target: reference,
metadata: None,
items: BTreeMap::new(),
inputs: BTreeMap::new(),
diagnostics: Vec::new(),
provenance: observation.clone().into_provenance(None, None),
};
if let Some(input) = read_input(
repository,
package,
&package.metadata_path,
"package-metadata",
&mut result,
&mut observation,
) && let Some(text) = &input.text
{
result.metadata = metadata::parse(text, &input.source, &mut result.diagnostics);
}
(result, observation)
}
fn read_input(
repository: &ResolvedRepositoryPaths,
package: &ResolvedPackagePaths,
path: &Path,
kind: &str,
result: &mut JuliaExtraction,
observation: &mut ExtractionObservation,
) -> Option<JuliaInput> {
let declared = path.strip_prefix(&package.path).ok()?;
let resolved = match resolve_input_file(&package.path, "julia.input", declared, &package.path) {
Ok(path) => path,
Err(error) => {
let mut diagnostic = error.to_diagnostic("julia-input".try_into().unwrap());
diagnostic.source = None;
diagnostic.related_entity = None;
result.diagnostics.push(diagnostic);
return None;
}
};
if crate::generated_storage::is_generated_file(&resolved) {
result.diagnostics.push(Diagnostic::new(
DiagnosticCode::JuliaSourceRead,
Severity::Error,
"Generated storage cannot be a Julia source input.",
));
return None;
}
let location = repository.source_location(&resolved, None).ok()?;
if let Some(input) = result.inputs.get(&location.path) {
return Some(input.clone());
}
let bytes = match fs::read(&resolved) {
Ok(bytes) => bytes,
Err(error) => {
result.diagnostics.push(diagnostic(
DiagnosticCode::JuliaSourceRead,
Severity::Error,
format!("Cannot read Julia input: {:?}.", error.kind()),
&location,
));
return None;
}
};
let fingerprint = fingerprint_bytes(&bytes);
let (text, raw_bytes) = match String::from_utf8(bytes) {
Ok(text) if text.len() <= u32::MAX as usize => (Some(text), None),
Ok(_) => {
result.diagnostics.push(diagnostic(
DiagnosticCode::JuliaSourceRead,
Severity::Error,
"Julia input exceeds Fatou's byte-range limit.",
&location,
));
return None;
}
Err(error) => {
result.diagnostics.push(diagnostic(
DiagnosticCode::JuliaSourceRead,
Severity::Error,
"Julia inputs must use UTF-8 encoding.",
&location,
));
(None, Some(error.into_bytes()))
}
};
let parsers = if text.is_some() {
if kind == "package-metadata" {
observation
.parsers
.entry("toml".into())
.or_insert(ParserProvenance {
version: env!("DIPLODOCUS_TOML_VERSION").into(),
role: "julia-metadata".into(),
settings: BTreeMap::new(),
});
BTreeSet::from(["toml".into()])
} else {
observation
.parsers
.entry("fatou-parser".into())
.or_insert(ParserProvenance {
version: FATOU_VERSION.into(),
role: "julia-source-and-documentation".into(),
settings: BTreeMap::from([
("grammar".into(), "julia-superset".into()),
("includes".into(), "unconditional-literal".into()),
("surface".into(), "public-or-documented".into()),
("documentation".into(), "julia-markdown-inert".into()),
]),
});
observation
.parsers
.entry("rowan".into())
.or_insert(ParserProvenance {
version: "0.17.0".into(),
role: "syntax-tree-and-byte-ranges".into(),
settings: BTreeMap::new(),
});
BTreeSet::from(["fatou-parser".into(), "rowan".into()])
}
} else {
BTreeSet::new()
};
observation
.inputs
.entry(location.repository.clone())
.or_default()
.insert(
location.path.clone(),
ExtractionInput {
kind: kind.into(),
fingerprint,
parsers,
},
);
let input = JuliaInput {
kind: kind.into(),
source: location,
text,
raw_bytes,
};
result
.inputs
.insert(input.source.path.clone(), input.clone());
Some(input)
}
pub(crate) fn diagnostic(
code: DiagnosticCode,
severity: Severity,
message: impl Into<String>,
source: &SourceLocation,
) -> Diagnostic {
let mut diagnostic =
Diagnostic::new(code, severity, message).with_source(DiagnosticSource::Repository {
repository: source.repository.clone(),
path: source.path.clone(),
});
diagnostic.span = source.span;
diagnostic
}
pub(crate) fn located(source: &SourceLocation, range: rowan::TextRange) -> SourceLocation {
SourceLocation {
span: Some(span(range)),
..source.clone()
}
}
pub(crate) fn span(range: rowan::TextRange) -> SourceSpan {
SourceSpan {
start: u32::from(range.start()) as usize,
end: u32::from(range.end()) as usize,
}
}
pub(crate) fn evidence(source: &SourceLocation, role: SourceRole) -> SourceEvidence {
SourceEvidence {
source: source.clone(),
role,
parsers: BTreeSet::from(["fatou-parser".into(), "rowan".into()]),
}
}
pub(crate) fn declaration_provenance(source: &SourceLocation) -> Provenance {
Provenance {
activity: ProvenanceActivity::Declaration,
source: Some(DiagnosticSource::Repository {
repository: source.repository.clone(),
path: source.path.clone(),
}),
span: source.span,
tools: BTreeMap::from([
("fatou-parser".into(), FATOU_VERSION.into()),
("rowan".into(), "0.17.0".into()),
("julia".into(), env!("CARGO_PKG_VERSION").into()),
]),
}
}
pub(crate) fn method_selector(text: &str) -> Option<(String, SignatureExpression)> {
signatures::selector(text)
}
fn normalized_path(path: &Path) -> PathBuf {
use std::path::Component;
let mut result = PathBuf::new();
for component in path.components() {
match component {
Component::CurDir => {}
Component::ParentDir => {
result.pop();
}
other => result.push(other.as_os_str()),
}
}
result
}