raf_source/parse.rs
1// SPDX-FileCopyrightText: 2026 Blackcat InformaticsĀ® Inc. <paudley@blackcatinformatics.ca>
2// SPDX-License-Identifier: MIT OR Apache-2.0 OR MulanPSL-2.0
3
4//! Bounded parsing: a pre-scan, then `syn` on a dedicated thread.
5
6use std::panic;
7use std::thread;
8
9use crate::budget::Budget;
10use crate::error::{LoadError, LoadErrorKind};
11use crate::meter::{ConfigurationUsage, FactAllowance, Meter, NativeAdmission, Usage};
12use crate::path::RelPath;
13use crate::prescan::{Scan, prescan_file, shebang};
14use crate::{ActiveConfiguration, LoadOptions, TreeError};
15
16mod entry;
17#[cfg(target_os = "linux")]
18pub(crate) mod parallel;
19
20/// The only worker output retained by the canonical walk.
21pub(crate) struct Extracted {
22 pub(crate) text: String,
23 pub(crate) head: ExtractedHead,
24 pub(crate) scan: Scan,
25}
26
27pub(crate) struct ExtractedHead {
28 pub(crate) facts: Result<(crate::facts::FileFacts, usize), ExtractionError>,
29 pub(crate) configuration_usage: Option<ConfigurationUsage>,
30}
31
32pub(crate) enum ExtractionError {
33 Configuration(LoadError),
34 Facts(crate::facts::ExtractError),
35}
36
37/// Admit declaration selection and facts only after the canonical walk grants
38/// the current file's budget. Diagnostic paths belong to that same reach.
39pub(crate) fn extract_head(
40 tree: &mut syn::File,
41 path: &RelPath,
42 text: &str,
43 allowance: FactAllowance,
44 configuration: Option<(&ActiveConfiguration, &LoadOptions, Usage)>,
45) -> ExtractedHead {
46 let Some((configuration, options, usage)) = configuration else {
47 return ExtractedHead {
48 facts: crate::facts::extract(tree, text, allowance).map_err(ExtractionError::Facts),
49 configuration_usage: None,
50 };
51 };
52 let local = Meter::resume(options, usage);
53 let facts = crate::configuration::filter(tree, configuration, &local, path)
54 .map_err(ExtractionError::Configuration)
55 .and_then(|()| {
56 crate::facts::extract(tree, text, allowance).map_err(ExtractionError::Facts)
57 });
58 ExtractedHead {
59 facts,
60 configuration_usage: Some(local.configuration_usage()),
61 }
62}
63
64/// A native storage refusal located at `file`.
65pub(crate) fn storage(file: &RelPath, source: purrdf_lex::allocation::StorageError) -> LoadError {
66 LoadError::new(file, None, LoadErrorKind::Storage(source))
67}
68
69/// Borrowed candidate spelling; materialization belongs to the admitted executor.
70#[derive(Clone, Copy)]
71pub(crate) enum Candidate<'a> {
72 Existing(&'a RelPath),
73 Joined {
74 base: &'a RelPath,
75 stem: Option<&'a str>,
76 input: &'a str,
77 suffix: &'a str,
78 },
79}
80
81#[cfg(target_os = "linux")]
82#[derive(Clone, Copy)]
83pub(crate) struct ConfigurationOrdinal(pub(crate) usize);
84
85/// A configuration admitted during canonical target selection.
86#[cfg(target_os = "linux")]
87#[derive(Clone, Copy)]
88pub(crate) enum ConfigurationKey {
89 Selected,
90 Locked(crate::PackageId, ConfigurationOrdinal),
91}
92
93/// The canonical reach's allowances and optional extraction context.
94#[cfg(target_os = "linux")]
95pub(crate) struct ExtractionGrant {
96 pub(crate) allowance: FactAllowance,
97 pub(crate) usage: Usage,
98 pub(crate) configuration: Option<ConfigurationKey>,
99}
100
101/// A coordinator-only parse executor, with no shared Tree or meter.
102pub(crate) trait Parser {
103 /// Whether another speculative candidate can be inspected now.
104 fn has_capacity(&self) -> bool;
105 fn resize_frontier(
106 &self,
107 previous: usize,
108 next: usize,
109 ) -> Result<(), purrdf_lex::allocation::StorageError>;
110 /// False leaves this candidate for a future frontier advance.
111 fn speculate(
112 &self,
113 tree: &dyn crate::Tree,
114 candidate: Candidate<'_>,
115 options: &LoadOptions,
116 seen: &dyn Fn(&crate::FileId) -> bool,
117 ) -> bool;
118 #[cfg(target_os = "linux")]
119 fn parse(
120 &self,
121 path: &RelPath,
122 id: &crate::FileId,
123 text: String,
124 options: &LoadOptions,
125 grant: ExtractionGrant,
126 ) -> Result<Extracted, LoadError>;
127}
128
129/// The stack of the thread that parses, in bytes: 64 MiB.
130///
131/// The caller's stack is not used, so a deep file costs this thread and not
132/// the thread of a test or of a tool.
133pub const PARSE_STACK_BYTES: usize = 64 * 1024 * 1024;
134
135/// An immutable source admitted by the caller-thread guard scan.
136///
137/// Private fields bind the handle to its borrowed path and text. Reading counts
138/// starts no worker or native lexer. Its consuming parse methods use that
139/// same text without repeating the guard scan. Admission is not syntax
140/// validation and does not establish a complete native heap or work bound.
141#[derive(Debug)]
142pub struct PreparedSource<'source> {
143 file: &'source RelPath,
144 text: &'source str,
145 scan: Scan,
146 native_bound: usize,
147}
148
149/// Guarded source lexed on its existing scoped worker, before syntax parsing.
150///
151/// Private fields bind the path, ignored header and original native tokens.
152/// The ticket is available only inside [`PreparedSource::with_lexed`]. Its
153/// consuming parse methods use that same stream on the current worker; they
154/// perform no additional guard scan, lex or worker startup.
155#[derive(Debug)]
156pub struct LexedSource<'source> {
157 file: &'source RelPath,
158 original: proc_macro2::TokenStream,
159 header: Option<String>,
160}
161
162impl<'source> LexedSource<'source> {
163 fn new(file: &'source RelPath, text: &str) -> Result<Self, LoadError> {
164 let (original, header) = lex_source(file, text)?;
165 Ok(Self {
166 file,
167 original,
168 header,
169 })
170 }
171
172 /// Borrow the original tokens before the native syntax parser starts.
173 ///
174 /// Cloning or iterating native tokens can allocate. Callers must admit
175 /// measurement itself before using this view to admit later parser work.
176 #[must_use]
177 pub const fn tokens(&self) -> &proc_macro2::TokenStream {
178 &self.original
179 }
180
181 /// Parse the ticket's stream and lend the syntax on the current worker.
182 ///
183 /// # Errors
184 /// Returns located native syntax errors before the visitor runs.
185 ///
186 /// # Panics
187 /// Parser and visitor panics unwind this worker; the enclosing prepared
188 /// operation resumes their original payload on its caller after joining.
189 pub fn parse<R>(self, visit: impl FnOnce(&syn::File) -> R) -> Result<R, LoadError> {
190 self.parse_mut(|tree| visit(tree))
191 }
192
193 /// Parse once and lend the syntax with its original native tokens.
194 ///
195 /// # Errors
196 /// Returns located native syntax errors before the visitor runs.
197 ///
198 /// # Panics
199 /// Parser and visitor panics follow the enclosing worker's joined path.
200 pub fn parse_with_tokens<R>(
201 self,
202 visit: impl FnOnce(&syn::File, proc_macro2::TokenStream) -> R,
203 ) -> Result<R, LoadError> {
204 let Self {
205 file,
206 original,
207 header,
208 } = self;
209 let tree = syntax(file, original.clone(), header)?;
210 Ok(visit(&tree, original))
211 }
212
213 fn into_syntax(self) -> Result<syn::File, LoadError> {
214 syntax(self.file, self.original, self.header)
215 }
216
217 fn parse_mut<R>(self, visit: impl FnOnce(&mut syn::File) -> R) -> Result<R, LoadError> {
218 self.into_syntax().map(|mut tree| visit(&mut tree))
219 }
220}
221
222/// Check the default file-token and recursion guards without starting a worker.
223///
224/// The returned handle binds the counts and subsequent parsing to immutable
225/// borrowed input. It performs no filesystem access or source mutation.
226///
227/// # Errors
228/// Returns the located guard refusal, or storage-size overflow when the native
229/// token ceiling cannot be represented. Syntax is checked only when parsing.
230pub fn prepare_source<'source>(
231 file: &'source RelPath,
232 text: &'source str,
233) -> Result<PreparedSource<'source>, LoadError> {
234 PreparedSource::new(file, text, Budget::FileTokens.limit())
235}
236
237impl<'source> PreparedSource<'source> {
238 fn new(
239 file: &'source RelPath,
240 text: &'source str,
241 max_tokens: usize,
242 ) -> Result<Self, LoadError> {
243 let scan = prescan_file(file, text, max_tokens)?;
244 let native_bound = scan
245 .tokens
246 .checked_mul(3)
247 .and_then(|tokens| text.len().checked_mul(2).map(|bytes| tokens.min(bytes)))
248 .ok_or_else(|| {
249 LoadError::new(
250 file,
251 None,
252 LoadErrorKind::Storage(purrdf_lex::allocation::StorageError::SizeOverflow),
253 )
254 })?;
255 Ok(Self {
256 file,
257 text,
258 scan,
259 native_bound,
260 })
261 }
262
263 /// Units compared against the guard's file-token limit.
264 ///
265 /// Numeric guard runs can fuse native leaves; this is not a native-node count.
266 #[must_use]
267 pub const fn token_count(&self) -> usize {
268 self.scan.tokens
269 }
270
271 /// The highest additive recursion counter reached by the guard scan.
272 #[must_use]
273 pub const fn recursion(&self) -> usize {
274 self.scan.recursion
275 }
276
277 /// Upper bound on complete native token-tree nodes if lexing succeeds.
278 ///
279 /// At most three native nodes arise per guard unit; source bytes separately
280 /// bound documentation lowering at twice their length. This is the smaller
281 /// checked bound and does not bound native AST storage or parser work.
282 #[must_use]
283 pub const fn native_token_bound(&self) -> usize {
284 self.native_bound
285 }
286
287 /// Lex on one scoped worker and lend a ticket before syntax parsing starts.
288 ///
289 /// The callback may admit parser resources from the actual token stream,
290 /// then consume the ticket. Measurement needs its own prior admission.
291 /// Lexing alone does not claim valid syntax or compiler acceptance. The
292 /// callback's typed result is retained unchanged; only its `Send` value
293 /// leaves the joined worker. No source code or Git state is changed.
294 ///
295 /// # Errors
296 /// Returns located native lexing, storage or worker-start errors. Syntax
297 /// errors are returned by the ticket's consuming parse methods.
298 ///
299 /// # Panics
300 /// Resumes parser or callback panics after joining, with the original payload.
301 pub fn with_lexed<R: Send>(
302 self,
303 visit: impl FnOnce(LexedSource<'source>) -> R + Send,
304 ) -> Result<R, LoadError> {
305 let Self { file, text, .. } = self;
306 on_worker(file, move || LexedSource::new(file, text).map(visit))
307 }
308
309 /// Parse the admitted text on its scoped worker, then join it.
310 ///
311 /// # Errors
312 /// Returns the native storage, syntax or worker-start errors of [`parse_source`].
313 ///
314 /// # Panics
315 /// Resumes parser or visitor panics on the caller with the original payload.
316 pub fn parse<R: Send>(
317 self,
318 visit: impl FnOnce(&syn::File) -> R + Send,
319 ) -> Result<R, LoadError> {
320 self.with_lexed(|lexed| lexed.parse(visit))?
321 }
322
323 /// Parse once and lend the original native tokens to the worker visitor.
324 ///
325 /// # Errors
326 /// Returns the native errors of [`parse_source_with_tokens`], without rescanning.
327 ///
328 /// # Panics
329 /// Resumes parser or visitor panics on the caller with the original payload.
330 pub fn parse_with_tokens<R: Send>(
331 self,
332 visit: impl FnOnce(&syn::File, proc_macro2::TokenStream) -> R + Send,
333 ) -> Result<R, LoadError> {
334 self.with_lexed(|lexed| lexed.parse_with_tokens(visit))?
335 }
336
337 fn parse_mut<R: Send>(
338 self,
339 visit: impl FnOnce(&mut syn::File) -> R + Send,
340 ) -> Result<R, LoadError> {
341 self.with_lexed(|lexed| lexed.parse_mut(visit))?
342 }
343}
344
345/// Parse `text` as a Rust file and run `visit` on the syntax tree.
346///
347/// The recursion budgets and the token budget, at its default limit
348/// ([`Budget::FileTokens`](crate::Budget::FileTokens)), are checked on the
349/// caller's thread first. The parse
350/// and `visit` then run on a scoped thread named after `file`, with a stack of
351/// [`PARSE_STACK_BYTES`]; a stack overflow in the runtime names that thread.
352/// A panic in `visit` or in the parser is raised again on the caller's thread.
353///
354/// The tree is dropped when `visit` returns, so `visit` copies out what the
355/// caller keeps.
356///
357/// # Errors
358///
359/// Returns a [`LoadError`] naming `file` when the text exceeds a budget
360/// ([`LoadErrorKind::Budget`]), is not valid Rust ([`LoadErrorKind::Syntax`])
361/// or the thread cannot start ([`LoadErrorKind::Spawn`]).
362///
363/// # Panics
364///
365/// Panics only when `visit` or the parser panics, with the original payload.
366pub fn parse_source<R: Send>(
367 file: &RelPath,
368 text: &str,
369 visit: impl FnOnce(&syn::File) -> R + Send,
370) -> Result<R, LoadError> {
371 prepare_source(file, text)?.parse(visit)
372}
373
374/// Parse once and give the visitor the syntax and its original native tokens.
375///
376/// The same file token and recursion checks as [`parse_source`] run before the
377/// worker starts. Both native values stay on that worker; only `Send` results
378/// return. The AST is parsed from one shared stream clone; native parsing can
379/// copy token payloads. The complete source is not lexed a second time, and
380/// caller-thread spans are never invalidated.
381///
382/// Byte ranges index the original `text`, including after a leading BOM or
383/// shebang. The BOM is handled by the native lexer. Language-ignored shebang
384/// bytes are replaced with same-length whitespace solely for lexical input;
385/// the original header is retained in `syn::File::shebang`. No file is edited
386/// and the header is never executed. Native parser/token/TLS heap and visitor
387/// allocations retain their separate resource contracts.
388///
389/// # Errors
390/// Returns the located native budget, storage, syntax or worker-start refusal.
391/// The visitor is never called after a failed admission, lex or parse.
392///
393/// # Panics
394/// Resumes parser or visitor panics on the caller with their original payload.
395pub fn parse_source_with_tokens<R: Send>(
396 file: &RelPath,
397 text: &str,
398 visit: impl FnOnce(&syn::File, proc_macro2::TokenStream) -> R + Send,
399) -> Result<R, LoadError> {
400 prepare_source(file, text)?.parse_with_tokens(visit)
401}
402
403/// One explicitly selected source and its owned syntax tree.
404///
405/// The syntax and its spans stay on the worker that parsed them. A batch visitor
406/// can move these records into its own analysis structures and return `Send`
407/// report data through [`parse_sources`].
408pub struct ParsedSource {
409 /// The caller's source identity; no discovery or module selection is performed.
410 pub path: RelPath,
411 /// The original source text, moved from the input without copying.
412 pub text: String,
413 /// The syntax tree, owned by the visitor.
414 pub syntax: syn::File,
415}
416
417/// Parse an explicit corpus once and give all its syntax trees to one visitor.
418///
419/// Input order and identities are preserved, including repeated identities.
420/// Before parsing any file, the complete input is checked against
421/// `options.max_file_bytes`, `options.max_total_bytes`, `options.max_file_tokens`
422/// and the native recursion budgets. The result-vector allocation is charged to
423/// `options.max_item_bytes`. This bounds source input and corpus-container storage;
424/// it does not meter the aggregate heap of retained `syn` trees or allocations
425/// made by `visit`. Each file retains the parser's per-file resource bounds.
426///
427/// Parsing, visiting and dropping the trees run on one scoped worker with
428/// [`PARSE_STACK_BYTES`]. The visitor runs exactly once, after every file parses,
429/// including for an empty corpus. Only its `Send` result leaves the worker.
430/// A visitor returning a `Result` retains its own typed error in that result.
431/// No filesystem, Cargo, target selection or duplicate policy is involved.
432///
433/// # Errors
434///
435/// Returns the native [`LoadError`] naming the input that exceeded a limit or
436/// failed to parse. Worker startup and corpus-vector allocation failures are
437/// also errors. No failed corpus reaches the visitor as a partial success.
438///
439/// # Panics
440///
441/// A parser or visitor panic resumes on the caller with its original payload.
442pub fn parse_sources<R: Send>(
443 inputs: Vec<(RelPath, String)>,
444 options: &LoadOptions,
445 visit: impl FnOnce(Vec<ParsedSource>) -> R + Send,
446) -> Result<R, LoadError> {
447 let meter = Meter::new(options);
448 for (file, text) in &inputs {
449 if text.len() as u64 > options.max_file_bytes {
450 return Err(LoadError::new(
451 file,
452 None,
453 LoadErrorKind::Tree(Box::new(TreeError::TooLarge {
454 path: file.to_string(),
455 limit: options.max_file_bytes,
456 })),
457 ));
458 }
459 meter.read(file, text)?;
460 prescan_file(file, text, options.max_file_tokens)?;
461 }
462 let file = inputs
463 .first()
464 .map_or_else(RelPath::root, |(file, _)| file.clone());
465 let worker_name = file.clone();
466 on_worker(&worker_name, move || {
467 let mut trees = Vec::new();
468 let mut admission = NativeAdmission::items(&meter, &file, None, 0);
469 purrdf_lex::allocation::Memory::new(&mut admission)
470 .reserve(&mut trees, inputs.len())
471 .map_err(|error| admission.graph_error(error.into()))?;
472 for (path, text) in inputs {
473 let syntax = parse_syntax(&path, &text)?;
474 trees.push(ParsedSource { path, text, syntax });
475 }
476 Ok(visit(trees))
477 })
478}
479
480/// [`parse_source`] with a token limit of `max_tokens`, also returning what
481/// the pre-scan counted.
482pub(crate) fn parse_counted<R: Send>(
483 file: &RelPath,
484 text: &str,
485 max_tokens: usize,
486 visit: impl FnOnce(&syn::File) -> R + Send,
487) -> Result<(R, Scan), LoadError> {
488 parse_counted_mut(file, text, max_tokens, |tree| visit(tree))
489}
490
491/// The same bounded worker, allowing declaration selection before fact extraction.
492pub(crate) fn parse_counted_mut<R: Send>(
493 file: &RelPath,
494 text: &str,
495 max_tokens: usize,
496 visit: impl FnOnce(&mut syn::File) -> R + Send,
497) -> Result<(R, Scan), LoadError> {
498 let prepared = PreparedSource::new(file, text, max_tokens)?;
499 let scan = prepared.scan;
500 prepared.parse_mut(visit).map(|result| (result, scan))
501}
502
503fn on_worker<R: Send>(
504 file: &RelPath,
505 work: impl FnOnce() -> Result<R, LoadError> + Send,
506) -> Result<R, LoadError> {
507 thread::scope(|scope| {
508 let worker = thread::Builder::new()
509 .name(file.to_string())
510 .stack_size(PARSE_STACK_BYTES)
511 .spawn_scoped(scope, move || {
512 entry::claim_registry(file)?;
513 work()
514 })
515 .map_err(|source| LoadError::new(file, None, LoadErrorKind::Spawn(source)))?;
516 match worker.join() {
517 Ok(result) => result,
518 Err(payload) => panic::resume_unwind(payload),
519 }
520 })
521}
522
523fn parse_syntax(file: &RelPath, text: &str) -> Result<syn::File, LoadError> {
524 LexedSource::new(file, text)?.into_syntax()
525}
526
527fn lex_source(
528 file: &RelPath,
529 text: &str,
530) -> Result<(proc_macro2::TokenStream, Option<String>), LoadError> {
531 use purrdf_lex::allocation::{Memory, Resident};
532 let mut resident = Resident;
533 let mut memory = Memory::new(&mut resident);
534 let mut header = None;
535 let normalized = if let Some(range) = shebang(text) {
536 let error = |source| LoadError::new(file, Some(1), LoadErrorKind::Storage(source));
537 header = Some(memory.string(&text[range.clone()]).map_err(error)?);
538 // Padding preserves byte positions and all newlines. There are no
539 // syntax positions within the ignored first-line header itself.
540 std::borrow::Cow::Owned(
541 memory
542 .format(&format_args!(
543 "{}{blank:width$}{}",
544 &text[..range.start],
545 &text[range.end..],
546 blank = "",
547 width = range.len()
548 ))
549 .map_err(error)?,
550 )
551 } else {
552 std::borrow::Cow::Borrowed(text)
553 };
554 let original = normalized
555 .parse::<proc_macro2::TokenStream>()
556 .map_err(|error| located_syntax(file, error.span(), error.to_string()))?;
557 Ok((original, header))
558}
559
560fn syntax(
561 file: &RelPath,
562 original: proc_macro2::TokenStream,
563 header: Option<String>,
564) -> Result<syn::File, LoadError> {
565 #[cfg(all(test, target_os = "linux"))]
566 let _parsing = parallel::tests::parsing(file);
567 let mut tree: syn::File = syn::parse2(original)
568 .map_err(|error| located_syntax(file, error.span(), error.to_string()))?;
569 tree.shebang = header;
570 Ok(tree)
571}
572
573fn located_syntax(file: &RelPath, span: proc_macro2::Span, message: String) -> LoadError {
574 // Read thread-local span data before returning the error to the caller.
575 let start = span.start();
576 LoadError::new(
577 file,
578 Some(start.line),
579 LoadErrorKind::Syntax {
580 message,
581 column: start.column + 1,
582 },
583 )
584}