use std::panic;
use std::thread;
use crate::budget::Budget;
use crate::error::{LoadError, LoadErrorKind};
use crate::meter::{ConfigurationUsage, FactAllowance, Meter, NativeAdmission, Usage};
use crate::path::RelPath;
use crate::prescan::{Scan, prescan_file, shebang};
use crate::{ActiveConfiguration, LoadOptions, TreeError};
mod entry;
#[cfg(target_os = "linux")]
pub(crate) mod parallel;
pub(crate) struct Extracted {
pub(crate) text: String,
pub(crate) head: ExtractedHead,
pub(crate) scan: Scan,
}
pub(crate) struct ExtractedHead {
pub(crate) facts: Result<(crate::facts::FileFacts, usize), ExtractionError>,
pub(crate) configuration_usage: Option<ConfigurationUsage>,
}
pub(crate) enum ExtractionError {
Configuration(LoadError),
Facts(crate::facts::ExtractError),
}
pub(crate) fn extract_head(
tree: &mut syn::File,
path: &RelPath,
text: &str,
allowance: FactAllowance,
configuration: Option<(&ActiveConfiguration, &LoadOptions, Usage)>,
) -> ExtractedHead {
let Some((configuration, options, usage)) = configuration else {
return ExtractedHead {
facts: crate::facts::extract(tree, text, allowance).map_err(ExtractionError::Facts),
configuration_usage: None,
};
};
let local = Meter::resume(options, usage);
let facts = crate::configuration::filter(tree, configuration, &local, path)
.map_err(ExtractionError::Configuration)
.and_then(|()| {
crate::facts::extract(tree, text, allowance).map_err(ExtractionError::Facts)
});
ExtractedHead {
facts,
configuration_usage: Some(local.configuration_usage()),
}
}
pub(crate) fn storage(file: &RelPath, source: purrdf_lex::allocation::StorageError) -> LoadError {
LoadError::new(file, None, LoadErrorKind::Storage(source))
}
#[derive(Clone, Copy)]
pub(crate) enum Candidate<'a> {
Existing(&'a RelPath),
Joined {
base: &'a RelPath,
stem: Option<&'a str>,
input: &'a str,
suffix: &'a str,
},
}
#[cfg(target_os = "linux")]
#[derive(Clone, Copy)]
pub(crate) struct ConfigurationOrdinal(pub(crate) usize);
#[cfg(target_os = "linux")]
#[derive(Clone, Copy)]
pub(crate) enum ConfigurationKey {
Selected,
Locked(crate::PackageId, ConfigurationOrdinal),
}
#[cfg(target_os = "linux")]
pub(crate) struct ExtractionGrant {
pub(crate) allowance: FactAllowance,
pub(crate) usage: Usage,
pub(crate) configuration: Option<ConfigurationKey>,
}
pub(crate) trait Parser {
fn has_capacity(&self) -> bool;
fn resize_frontier(
&self,
previous: usize,
next: usize,
) -> Result<(), purrdf_lex::allocation::StorageError>;
fn speculate(
&self,
tree: &dyn crate::Tree,
candidate: Candidate<'_>,
options: &LoadOptions,
seen: &dyn Fn(&crate::FileId) -> bool,
) -> bool;
#[cfg(target_os = "linux")]
fn parse(
&self,
path: &RelPath,
id: &crate::FileId,
text: String,
options: &LoadOptions,
grant: ExtractionGrant,
) -> Result<Extracted, LoadError>;
}
pub const PARSE_STACK_BYTES: usize = 64 * 1024 * 1024;
#[derive(Debug)]
pub struct PreparedSource<'source> {
file: &'source RelPath,
text: &'source str,
scan: Scan,
native_bound: usize,
}
#[derive(Debug)]
pub struct LexedSource<'source> {
file: &'source RelPath,
original: proc_macro2::TokenStream,
header: Option<String>,
}
impl<'source> LexedSource<'source> {
fn new(file: &'source RelPath, text: &str) -> Result<Self, LoadError> {
let (original, header) = lex_source(file, text)?;
Ok(Self {
file,
original,
header,
})
}
#[must_use]
pub const fn tokens(&self) -> &proc_macro2::TokenStream {
&self.original
}
pub fn parse<R>(self, visit: impl FnOnce(&syn::File) -> R) -> Result<R, LoadError> {
self.parse_mut(|tree| visit(tree))
}
pub fn parse_with_tokens<R>(
self,
visit: impl FnOnce(&syn::File, proc_macro2::TokenStream) -> R,
) -> Result<R, LoadError> {
let Self {
file,
original,
header,
} = self;
let tree = syntax(file, original.clone(), header)?;
Ok(visit(&tree, original))
}
fn into_syntax(self) -> Result<syn::File, LoadError> {
syntax(self.file, self.original, self.header)
}
fn parse_mut<R>(self, visit: impl FnOnce(&mut syn::File) -> R) -> Result<R, LoadError> {
self.into_syntax().map(|mut tree| visit(&mut tree))
}
}
pub fn prepare_source<'source>(
file: &'source RelPath,
text: &'source str,
) -> Result<PreparedSource<'source>, LoadError> {
PreparedSource::new(file, text, Budget::FileTokens.limit())
}
impl<'source> PreparedSource<'source> {
fn new(
file: &'source RelPath,
text: &'source str,
max_tokens: usize,
) -> Result<Self, LoadError> {
let scan = prescan_file(file, text, max_tokens)?;
let native_bound = scan
.tokens
.checked_mul(3)
.and_then(|tokens| text.len().checked_mul(2).map(|bytes| tokens.min(bytes)))
.ok_or_else(|| {
LoadError::new(
file,
None,
LoadErrorKind::Storage(purrdf_lex::allocation::StorageError::SizeOverflow),
)
})?;
Ok(Self {
file,
text,
scan,
native_bound,
})
}
#[must_use]
pub const fn token_count(&self) -> usize {
self.scan.tokens
}
#[must_use]
pub const fn recursion(&self) -> usize {
self.scan.recursion
}
#[must_use]
pub const fn native_token_bound(&self) -> usize {
self.native_bound
}
pub fn with_lexed<R: Send>(
self,
visit: impl FnOnce(LexedSource<'source>) -> R + Send,
) -> Result<R, LoadError> {
let Self { file, text, .. } = self;
on_worker(file, move || LexedSource::new(file, text).map(visit))
}
pub fn parse<R: Send>(
self,
visit: impl FnOnce(&syn::File) -> R + Send,
) -> Result<R, LoadError> {
self.with_lexed(|lexed| lexed.parse(visit))?
}
pub fn parse_with_tokens<R: Send>(
self,
visit: impl FnOnce(&syn::File, proc_macro2::TokenStream) -> R + Send,
) -> Result<R, LoadError> {
self.with_lexed(|lexed| lexed.parse_with_tokens(visit))?
}
fn parse_mut<R: Send>(
self,
visit: impl FnOnce(&mut syn::File) -> R + Send,
) -> Result<R, LoadError> {
self.with_lexed(|lexed| lexed.parse_mut(visit))?
}
}
pub fn parse_source<R: Send>(
file: &RelPath,
text: &str,
visit: impl FnOnce(&syn::File) -> R + Send,
) -> Result<R, LoadError> {
prepare_source(file, text)?.parse(visit)
}
pub fn parse_source_with_tokens<R: Send>(
file: &RelPath,
text: &str,
visit: impl FnOnce(&syn::File, proc_macro2::TokenStream) -> R + Send,
) -> Result<R, LoadError> {
prepare_source(file, text)?.parse_with_tokens(visit)
}
pub struct ParsedSource {
pub path: RelPath,
pub text: String,
pub syntax: syn::File,
}
pub fn parse_sources<R: Send>(
inputs: Vec<(RelPath, String)>,
options: &LoadOptions,
visit: impl FnOnce(Vec<ParsedSource>) -> R + Send,
) -> Result<R, LoadError> {
let meter = Meter::new(options);
for (file, text) in &inputs {
if text.len() as u64 > options.max_file_bytes {
return Err(LoadError::new(
file,
None,
LoadErrorKind::Tree(Box::new(TreeError::TooLarge {
path: file.to_string(),
limit: options.max_file_bytes,
})),
));
}
meter.read(file, text)?;
prescan_file(file, text, options.max_file_tokens)?;
}
let file = inputs
.first()
.map_or_else(RelPath::root, |(file, _)| file.clone());
let worker_name = file.clone();
on_worker(&worker_name, move || {
let mut trees = Vec::new();
let mut admission = NativeAdmission::items(&meter, &file, None, 0);
purrdf_lex::allocation::Memory::new(&mut admission)
.reserve(&mut trees, inputs.len())
.map_err(|error| admission.graph_error(error.into()))?;
for (path, text) in inputs {
let syntax = parse_syntax(&path, &text)?;
trees.push(ParsedSource { path, text, syntax });
}
Ok(visit(trees))
})
}
pub(crate) fn parse_counted<R: Send>(
file: &RelPath,
text: &str,
max_tokens: usize,
visit: impl FnOnce(&syn::File) -> R + Send,
) -> Result<(R, Scan), LoadError> {
parse_counted_mut(file, text, max_tokens, |tree| visit(tree))
}
pub(crate) fn parse_counted_mut<R: Send>(
file: &RelPath,
text: &str,
max_tokens: usize,
visit: impl FnOnce(&mut syn::File) -> R + Send,
) -> Result<(R, Scan), LoadError> {
let prepared = PreparedSource::new(file, text, max_tokens)?;
let scan = prepared.scan;
prepared.parse_mut(visit).map(|result| (result, scan))
}
fn on_worker<R: Send>(
file: &RelPath,
work: impl FnOnce() -> Result<R, LoadError> + Send,
) -> Result<R, LoadError> {
thread::scope(|scope| {
let worker = thread::Builder::new()
.name(file.to_string())
.stack_size(PARSE_STACK_BYTES)
.spawn_scoped(scope, move || {
entry::claim_registry(file)?;
work()
})
.map_err(|source| LoadError::new(file, None, LoadErrorKind::Spawn(source)))?;
match worker.join() {
Ok(result) => result,
Err(payload) => panic::resume_unwind(payload),
}
})
}
fn parse_syntax(file: &RelPath, text: &str) -> Result<syn::File, LoadError> {
LexedSource::new(file, text)?.into_syntax()
}
fn lex_source(
file: &RelPath,
text: &str,
) -> Result<(proc_macro2::TokenStream, Option<String>), LoadError> {
use purrdf_lex::allocation::{Memory, Resident};
let mut resident = Resident;
let mut memory = Memory::new(&mut resident);
let mut header = None;
let normalized = if let Some(range) = shebang(text) {
let error = |source| LoadError::new(file, Some(1), LoadErrorKind::Storage(source));
header = Some(memory.string(&text[range.clone()]).map_err(error)?);
std::borrow::Cow::Owned(
memory
.format(&format_args!(
"{}{blank:width$}{}",
&text[..range.start],
&text[range.end..],
blank = "",
width = range.len()
))
.map_err(error)?,
)
} else {
std::borrow::Cow::Borrowed(text)
};
let original = normalized
.parse::<proc_macro2::TokenStream>()
.map_err(|error| located_syntax(file, error.span(), error.to_string()))?;
Ok((original, header))
}
fn syntax(
file: &RelPath,
original: proc_macro2::TokenStream,
header: Option<String>,
) -> Result<syn::File, LoadError> {
#[cfg(all(test, target_os = "linux"))]
let _parsing = parallel::tests::parsing(file);
let mut tree: syn::File = syn::parse2(original)
.map_err(|error| located_syntax(file, error.span(), error.to_string()))?;
tree.shebang = header;
Ok(tree)
}
fn located_syntax(file: &RelPath, span: proc_macro2::Span, message: String) -> LoadError {
let start = span.start();
LoadError::new(
file,
Some(start.line),
LoadErrorKind::Syntax {
message,
column: start.column + 1,
},
)
}