Skip to main content

raf_source/
parse.rs

1// SPDX-FileCopyrightText: 2026 Blackcat InformaticsĀ® Inc. <paudley@blackcatinformatics.ca>
2// SPDX-License-Identifier: MIT OR Apache-2.0 OR MulanPSL-2.0
3
4//! Bounded parsing: a pre-scan, then `syn` on a dedicated thread.
5
6use std::panic;
7use std::thread;
8
9use crate::budget::Budget;
10use crate::error::{LoadError, LoadErrorKind};
11use crate::meter::{ConfigurationUsage, FactAllowance, Meter, NativeAdmission, Usage};
12use crate::path::RelPath;
13use crate::prescan::{Scan, prescan_file, shebang};
14use crate::{ActiveConfiguration, LoadOptions, TreeError};
15
16mod entry;
17#[cfg(target_os = "linux")]
18pub(crate) mod parallel;
19
20/// The only worker output retained by the canonical walk.
21pub(crate) struct Extracted {
22    pub(crate) text: String,
23    pub(crate) head: ExtractedHead,
24    pub(crate) scan: Scan,
25}
26
27pub(crate) struct ExtractedHead {
28    pub(crate) facts: Result<(crate::facts::FileFacts, usize), ExtractionError>,
29    pub(crate) configuration_usage: Option<ConfigurationUsage>,
30}
31
32pub(crate) enum ExtractionError {
33    Configuration(LoadError),
34    Facts(crate::facts::ExtractError),
35}
36
37/// Admit declaration selection and facts only after the canonical walk grants
38/// the current file's budget. Diagnostic paths belong to that same reach.
39pub(crate) fn extract_head(
40    tree: &mut syn::File,
41    path: &RelPath,
42    text: &str,
43    allowance: FactAllowance,
44    configuration: Option<(&ActiveConfiguration, &LoadOptions, Usage)>,
45) -> ExtractedHead {
46    let Some((configuration, options, usage)) = configuration else {
47        return ExtractedHead {
48            facts: crate::facts::extract(tree, text, allowance).map_err(ExtractionError::Facts),
49            configuration_usage: None,
50        };
51    };
52    let local = Meter::resume(options, usage);
53    let facts = crate::configuration::filter(tree, configuration, &local, path)
54        .map_err(ExtractionError::Configuration)
55        .and_then(|()| {
56            crate::facts::extract(tree, text, allowance).map_err(ExtractionError::Facts)
57        });
58    ExtractedHead {
59        facts,
60        configuration_usage: Some(local.configuration_usage()),
61    }
62}
63
64/// A native storage refusal located at `file`.
65pub(crate) fn storage(file: &RelPath, source: purrdf_lex::allocation::StorageError) -> LoadError {
66    LoadError::new(file, None, LoadErrorKind::Storage(source))
67}
68
69/// Borrowed candidate spelling; materialization belongs to the admitted executor.
70#[derive(Clone, Copy)]
71pub(crate) enum Candidate<'a> {
72    Existing(&'a RelPath),
73    Joined {
74        base: &'a RelPath,
75        stem: Option<&'a str>,
76        input: &'a str,
77        suffix: &'a str,
78    },
79}
80
81#[cfg(target_os = "linux")]
82#[derive(Clone, Copy)]
83pub(crate) struct ConfigurationOrdinal(pub(crate) usize);
84
85/// A configuration admitted during canonical target selection.
86#[cfg(target_os = "linux")]
87#[derive(Clone, Copy)]
88pub(crate) enum ConfigurationKey {
89    Selected,
90    Locked(crate::PackageId, ConfigurationOrdinal),
91}
92
93/// The canonical reach's allowances and optional extraction context.
94#[cfg(target_os = "linux")]
95pub(crate) struct ExtractionGrant {
96    pub(crate) allowance: FactAllowance,
97    pub(crate) usage: Usage,
98    pub(crate) configuration: Option<ConfigurationKey>,
99}
100
101/// A coordinator-only parse executor, with no shared Tree or meter.
102pub(crate) trait Parser {
103    /// Whether another speculative candidate can be inspected now.
104    fn has_capacity(&self) -> bool;
105    fn resize_frontier(
106        &self,
107        previous: usize,
108        next: usize,
109    ) -> Result<(), purrdf_lex::allocation::StorageError>;
110    /// False leaves this candidate for a future frontier advance.
111    fn speculate(
112        &self,
113        tree: &dyn crate::Tree,
114        candidate: Candidate<'_>,
115        options: &LoadOptions,
116        seen: &dyn Fn(&crate::FileId) -> bool,
117    ) -> bool;
118    #[cfg(target_os = "linux")]
119    fn parse(
120        &self,
121        path: &RelPath,
122        id: &crate::FileId,
123        text: String,
124        options: &LoadOptions,
125        grant: ExtractionGrant,
126    ) -> Result<Extracted, LoadError>;
127}
128
129/// The stack of the thread that parses, in bytes: 64 MiB.
130///
131/// The caller's stack is not used, so a deep file costs this thread and not
132/// the thread of a test or of a tool.
133pub const PARSE_STACK_BYTES: usize = 64 * 1024 * 1024;
134
135/// An immutable source admitted by the caller-thread guard scan.
136///
137/// Private fields bind the handle to its borrowed path and text. Reading counts
138/// starts no worker or native lexer. Its consuming parse methods use that
139/// same text without repeating the guard scan. Admission is not syntax
140/// validation and does not establish a complete native heap or work bound.
141#[derive(Debug)]
142pub struct PreparedSource<'source> {
143    file: &'source RelPath,
144    text: &'source str,
145    scan: Scan,
146    native_bound: usize,
147}
148
149/// Guarded source lexed on its existing scoped worker, before syntax parsing.
150///
151/// Private fields bind the path, ignored header and original native tokens.
152/// The ticket is available only inside [`PreparedSource::with_lexed`]. Its
153/// consuming parse methods use that same stream on the current worker; they
154/// perform no additional guard scan, lex or worker startup.
155#[derive(Debug)]
156pub struct LexedSource<'source> {
157    file: &'source RelPath,
158    original: proc_macro2::TokenStream,
159    header: Option<String>,
160}
161
162impl<'source> LexedSource<'source> {
163    fn new(file: &'source RelPath, text: &str) -> Result<Self, LoadError> {
164        let (original, header) = lex_source(file, text)?;
165        Ok(Self {
166            file,
167            original,
168            header,
169        })
170    }
171
172    /// Borrow the original tokens before the native syntax parser starts.
173    ///
174    /// Cloning or iterating native tokens can allocate. Callers must admit
175    /// measurement itself before using this view to admit later parser work.
176    #[must_use]
177    pub const fn tokens(&self) -> &proc_macro2::TokenStream {
178        &self.original
179    }
180
181    /// Parse the ticket's stream and lend the syntax on the current worker.
182    ///
183    /// # Errors
184    /// Returns located native syntax errors before the visitor runs.
185    ///
186    /// # Panics
187    /// Parser and visitor panics unwind this worker; the enclosing prepared
188    /// operation resumes their original payload on its caller after joining.
189    pub fn parse<R>(self, visit: impl FnOnce(&syn::File) -> R) -> Result<R, LoadError> {
190        self.parse_mut(|tree| visit(tree))
191    }
192
193    /// Parse once and lend the syntax with its original native tokens.
194    ///
195    /// # Errors
196    /// Returns located native syntax errors before the visitor runs.
197    ///
198    /// # Panics
199    /// Parser and visitor panics follow the enclosing worker's joined path.
200    pub fn parse_with_tokens<R>(
201        self,
202        visit: impl FnOnce(&syn::File, proc_macro2::TokenStream) -> R,
203    ) -> Result<R, LoadError> {
204        let Self {
205            file,
206            original,
207            header,
208        } = self;
209        let tree = syntax(file, original.clone(), header)?;
210        Ok(visit(&tree, original))
211    }
212
213    fn into_syntax(self) -> Result<syn::File, LoadError> {
214        syntax(self.file, self.original, self.header)
215    }
216
217    fn parse_mut<R>(self, visit: impl FnOnce(&mut syn::File) -> R) -> Result<R, LoadError> {
218        self.into_syntax().map(|mut tree| visit(&mut tree))
219    }
220}
221
222/// Check the default file-token and recursion guards without starting a worker.
223///
224/// The returned handle binds the counts and subsequent parsing to immutable
225/// borrowed input. It performs no filesystem access or source mutation.
226///
227/// # Errors
228/// Returns the located guard refusal, or storage-size overflow when the native
229/// token ceiling cannot be represented. Syntax is checked only when parsing.
230pub fn prepare_source<'source>(
231    file: &'source RelPath,
232    text: &'source str,
233) -> Result<PreparedSource<'source>, LoadError> {
234    PreparedSource::new(file, text, Budget::FileTokens.limit())
235}
236
237impl<'source> PreparedSource<'source> {
238    fn new(
239        file: &'source RelPath,
240        text: &'source str,
241        max_tokens: usize,
242    ) -> Result<Self, LoadError> {
243        let scan = prescan_file(file, text, max_tokens)?;
244        let native_bound = scan
245            .tokens
246            .checked_mul(3)
247            .and_then(|tokens| text.len().checked_mul(2).map(|bytes| tokens.min(bytes)))
248            .ok_or_else(|| {
249                LoadError::new(
250                    file,
251                    None,
252                    LoadErrorKind::Storage(purrdf_lex::allocation::StorageError::SizeOverflow),
253                )
254            })?;
255        Ok(Self {
256            file,
257            text,
258            scan,
259            native_bound,
260        })
261    }
262
263    /// Units compared against the guard's file-token limit.
264    ///
265    /// Numeric guard runs can fuse native leaves; this is not a native-node count.
266    #[must_use]
267    pub const fn token_count(&self) -> usize {
268        self.scan.tokens
269    }
270
271    /// The highest additive recursion counter reached by the guard scan.
272    #[must_use]
273    pub const fn recursion(&self) -> usize {
274        self.scan.recursion
275    }
276
277    /// Upper bound on complete native token-tree nodes if lexing succeeds.
278    ///
279    /// At most three native nodes arise per guard unit; source bytes separately
280    /// bound documentation lowering at twice their length. This is the smaller
281    /// checked bound and does not bound native AST storage or parser work.
282    #[must_use]
283    pub const fn native_token_bound(&self) -> usize {
284        self.native_bound
285    }
286
287    /// Lex on one scoped worker and lend a ticket before syntax parsing starts.
288    ///
289    /// The callback may admit parser resources from the actual token stream,
290    /// then consume the ticket. Measurement needs its own prior admission.
291    /// Lexing alone does not claim valid syntax or compiler acceptance. The
292    /// callback's typed result is retained unchanged; only its `Send` value
293    /// leaves the joined worker. No source code or Git state is changed.
294    ///
295    /// # Errors
296    /// Returns located native lexing, storage or worker-start errors. Syntax
297    /// errors are returned by the ticket's consuming parse methods.
298    ///
299    /// # Panics
300    /// Resumes parser or callback panics after joining, with the original payload.
301    pub fn with_lexed<R: Send>(
302        self,
303        visit: impl FnOnce(LexedSource<'source>) -> R + Send,
304    ) -> Result<R, LoadError> {
305        let Self { file, text, .. } = self;
306        on_worker(file, move || LexedSource::new(file, text).map(visit))
307    }
308
309    /// Parse the admitted text on its scoped worker, then join it.
310    ///
311    /// # Errors
312    /// Returns the native storage, syntax or worker-start errors of [`parse_source`].
313    ///
314    /// # Panics
315    /// Resumes parser or visitor panics on the caller with the original payload.
316    pub fn parse<R: Send>(
317        self,
318        visit: impl FnOnce(&syn::File) -> R + Send,
319    ) -> Result<R, LoadError> {
320        self.with_lexed(|lexed| lexed.parse(visit))?
321    }
322
323    /// Parse once and lend the original native tokens to the worker visitor.
324    ///
325    /// # Errors
326    /// Returns the native errors of [`parse_source_with_tokens`], without rescanning.
327    ///
328    /// # Panics
329    /// Resumes parser or visitor panics on the caller with the original payload.
330    pub fn parse_with_tokens<R: Send>(
331        self,
332        visit: impl FnOnce(&syn::File, proc_macro2::TokenStream) -> R + Send,
333    ) -> Result<R, LoadError> {
334        self.with_lexed(|lexed| lexed.parse_with_tokens(visit))?
335    }
336
337    fn parse_mut<R: Send>(
338        self,
339        visit: impl FnOnce(&mut syn::File) -> R + Send,
340    ) -> Result<R, LoadError> {
341        self.with_lexed(|lexed| lexed.parse_mut(visit))?
342    }
343}
344
345/// Parse `text` as a Rust file and run `visit` on the syntax tree.
346///
347/// The recursion budgets and the token budget, at its default limit
348/// ([`Budget::FileTokens`](crate::Budget::FileTokens)), are checked on the
349/// caller's thread first. The parse
350/// and `visit` then run on a scoped thread named after `file`, with a stack of
351/// [`PARSE_STACK_BYTES`]; a stack overflow in the runtime names that thread.
352/// A panic in `visit` or in the parser is raised again on the caller's thread.
353///
354/// The tree is dropped when `visit` returns, so `visit` copies out what the
355/// caller keeps.
356///
357/// # Errors
358///
359/// Returns a [`LoadError`] naming `file` when the text exceeds a budget
360/// ([`LoadErrorKind::Budget`]), is not valid Rust ([`LoadErrorKind::Syntax`])
361/// or the thread cannot start ([`LoadErrorKind::Spawn`]).
362///
363/// # Panics
364///
365/// Panics only when `visit` or the parser panics, with the original payload.
366pub fn parse_source<R: Send>(
367    file: &RelPath,
368    text: &str,
369    visit: impl FnOnce(&syn::File) -> R + Send,
370) -> Result<R, LoadError> {
371    prepare_source(file, text)?.parse(visit)
372}
373
374/// Parse once and give the visitor the syntax and its original native tokens.
375///
376/// The same file token and recursion checks as [`parse_source`] run before the
377/// worker starts. Both native values stay on that worker; only `Send` results
378/// return. The AST is parsed from one shared stream clone; native parsing can
379/// copy token payloads. The complete source is not lexed a second time, and
380/// caller-thread spans are never invalidated.
381///
382/// Byte ranges index the original `text`, including after a leading BOM or
383/// shebang. The BOM is handled by the native lexer. Language-ignored shebang
384/// bytes are replaced with same-length whitespace solely for lexical input;
385/// the original header is retained in `syn::File::shebang`. No file is edited
386/// and the header is never executed. Native parser/token/TLS heap and visitor
387/// allocations retain their separate resource contracts.
388///
389/// # Errors
390/// Returns the located native budget, storage, syntax or worker-start refusal.
391/// The visitor is never called after a failed admission, lex or parse.
392///
393/// # Panics
394/// Resumes parser or visitor panics on the caller with their original payload.
395pub fn parse_source_with_tokens<R: Send>(
396    file: &RelPath,
397    text: &str,
398    visit: impl FnOnce(&syn::File, proc_macro2::TokenStream) -> R + Send,
399) -> Result<R, LoadError> {
400    prepare_source(file, text)?.parse_with_tokens(visit)
401}
402
403/// One explicitly selected source and its owned syntax tree.
404///
405/// The syntax and its spans stay on the worker that parsed them. A batch visitor
406/// can move these records into its own analysis structures and return `Send`
407/// report data through [`parse_sources`].
408pub struct ParsedSource {
409    /// The caller's source identity; no discovery or module selection is performed.
410    pub path: RelPath,
411    /// The original source text, moved from the input without copying.
412    pub text: String,
413    /// The syntax tree, owned by the visitor.
414    pub syntax: syn::File,
415}
416
417/// Parse an explicit corpus once and give all its syntax trees to one visitor.
418///
419/// Input order and identities are preserved, including repeated identities.
420/// Before parsing any file, the complete input is checked against
421/// `options.max_file_bytes`, `options.max_total_bytes`, `options.max_file_tokens`
422/// and the native recursion budgets. The result-vector allocation is charged to
423/// `options.max_item_bytes`. This bounds source input and corpus-container storage;
424/// it does not meter the aggregate heap of retained `syn` trees or allocations
425/// made by `visit`. Each file retains the parser's per-file resource bounds.
426///
427/// Parsing, visiting and dropping the trees run on one scoped worker with
428/// [`PARSE_STACK_BYTES`]. The visitor runs exactly once, after every file parses,
429/// including for an empty corpus. Only its `Send` result leaves the worker.
430/// A visitor returning a `Result` retains its own typed error in that result.
431/// No filesystem, Cargo, target selection or duplicate policy is involved.
432///
433/// # Errors
434///
435/// Returns the native [`LoadError`] naming the input that exceeded a limit or
436/// failed to parse. Worker startup and corpus-vector allocation failures are
437/// also errors. No failed corpus reaches the visitor as a partial success.
438///
439/// # Panics
440///
441/// A parser or visitor panic resumes on the caller with its original payload.
442pub fn parse_sources<R: Send>(
443    inputs: Vec<(RelPath, String)>,
444    options: &LoadOptions,
445    visit: impl FnOnce(Vec<ParsedSource>) -> R + Send,
446) -> Result<R, LoadError> {
447    let meter = Meter::new(options);
448    for (file, text) in &inputs {
449        if text.len() as u64 > options.max_file_bytes {
450            return Err(LoadError::new(
451                file,
452                None,
453                LoadErrorKind::Tree(Box::new(TreeError::TooLarge {
454                    path: file.to_string(),
455                    limit: options.max_file_bytes,
456                })),
457            ));
458        }
459        meter.read(file, text)?;
460        prescan_file(file, text, options.max_file_tokens)?;
461    }
462    let file = inputs
463        .first()
464        .map_or_else(RelPath::root, |(file, _)| file.clone());
465    let worker_name = file.clone();
466    on_worker(&worker_name, move || {
467        let mut trees = Vec::new();
468        let mut admission = NativeAdmission::items(&meter, &file, None, 0);
469        purrdf_lex::allocation::Memory::new(&mut admission)
470            .reserve(&mut trees, inputs.len())
471            .map_err(|error| admission.graph_error(error.into()))?;
472        for (path, text) in inputs {
473            let syntax = parse_syntax(&path, &text)?;
474            trees.push(ParsedSource { path, text, syntax });
475        }
476        Ok(visit(trees))
477    })
478}
479
480/// [`parse_source`] with a token limit of `max_tokens`, also returning what
481/// the pre-scan counted.
482pub(crate) fn parse_counted<R: Send>(
483    file: &RelPath,
484    text: &str,
485    max_tokens: usize,
486    visit: impl FnOnce(&syn::File) -> R + Send,
487) -> Result<(R, Scan), LoadError> {
488    parse_counted_mut(file, text, max_tokens, |tree| visit(tree))
489}
490
491/// The same bounded worker, allowing declaration selection before fact extraction.
492pub(crate) fn parse_counted_mut<R: Send>(
493    file: &RelPath,
494    text: &str,
495    max_tokens: usize,
496    visit: impl FnOnce(&mut syn::File) -> R + Send,
497) -> Result<(R, Scan), LoadError> {
498    let prepared = PreparedSource::new(file, text, max_tokens)?;
499    let scan = prepared.scan;
500    prepared.parse_mut(visit).map(|result| (result, scan))
501}
502
503fn on_worker<R: Send>(
504    file: &RelPath,
505    work: impl FnOnce() -> Result<R, LoadError> + Send,
506) -> Result<R, LoadError> {
507    thread::scope(|scope| {
508        let worker = thread::Builder::new()
509            .name(file.to_string())
510            .stack_size(PARSE_STACK_BYTES)
511            .spawn_scoped(scope, move || {
512                entry::claim_registry(file)?;
513                work()
514            })
515            .map_err(|source| LoadError::new(file, None, LoadErrorKind::Spawn(source)))?;
516        match worker.join() {
517            Ok(result) => result,
518            Err(payload) => panic::resume_unwind(payload),
519        }
520    })
521}
522
523fn parse_syntax(file: &RelPath, text: &str) -> Result<syn::File, LoadError> {
524    LexedSource::new(file, text)?.into_syntax()
525}
526
527fn lex_source(
528    file: &RelPath,
529    text: &str,
530) -> Result<(proc_macro2::TokenStream, Option<String>), LoadError> {
531    use purrdf_lex::allocation::{Memory, Resident};
532    let mut resident = Resident;
533    let mut memory = Memory::new(&mut resident);
534    let mut header = None;
535    let normalized = if let Some(range) = shebang(text) {
536        let error = |source| LoadError::new(file, Some(1), LoadErrorKind::Storage(source));
537        header = Some(memory.string(&text[range.clone()]).map_err(error)?);
538        // Padding preserves byte positions and all newlines. There are no
539        // syntax positions within the ignored first-line header itself.
540        std::borrow::Cow::Owned(
541            memory
542                .format(&format_args!(
543                    "{}{blank:width$}{}",
544                    &text[..range.start],
545                    &text[range.end..],
546                    blank = "",
547                    width = range.len()
548                ))
549                .map_err(error)?,
550        )
551    } else {
552        std::borrow::Cow::Borrowed(text)
553    };
554    let original = normalized
555        .parse::<proc_macro2::TokenStream>()
556        .map_err(|error| located_syntax(file, error.span(), error.to_string()))?;
557    Ok((original, header))
558}
559
560fn syntax(
561    file: &RelPath,
562    original: proc_macro2::TokenStream,
563    header: Option<String>,
564) -> Result<syn::File, LoadError> {
565    #[cfg(all(test, target_os = "linux"))]
566    let _parsing = parallel::tests::parsing(file);
567    let mut tree: syn::File = syn::parse2(original)
568        .map_err(|error| located_syntax(file, error.span(), error.to_string()))?;
569    tree.shebang = header;
570    Ok(tree)
571}
572
573fn located_syntax(file: &RelPath, span: proc_macro2::Span, message: String) -> LoadError {
574    // Read thread-local span data before returning the error to the caller.
575    let start = span.start();
576    LoadError::new(
577        file,
578        Some(start.line),
579        LoadErrorKind::Syntax {
580            message,
581            column: start.column + 1,
582        },
583    )
584}