vyre_libs/parsing/c/preprocess/
mod.rs

1//! C11 preprocessor passes.
2
3use crate::parsing::c::lex::tokens::{
4    TOK_PP_DEFINE, TOK_PP_ELIF, TOK_PP_ELIFDEF, TOK_PP_ELIFNDEF, TOK_PP_ELSE, TOK_PP_EMBED,
5    TOK_PP_ENDIF, TOK_PP_ERROR, TOK_PP_IDENT, TOK_PP_IF, TOK_PP_IFDEF, TOK_PP_IFNDEF,
6    TOK_PP_IMPORT, TOK_PP_INCLUDE, TOK_PP_INCLUDE_NEXT, TOK_PP_LINE, TOK_PP_NULL, TOK_PP_PRAGMA,
7    TOK_PP_SCCS, TOK_PP_UNDEF, TOK_PP_WARNING,
8};
9
10/// Shared CPU directive-row walk and per-row splice/classify/payload scan.
11mod directive_scan;
12use directive_scan::ScannedDirective;
13
14/// Preprocessor side-effect metadata.
15pub mod effects;
16/// Macro-expansion kernel.
17pub mod expansion;
18/// GPU char-constant scanner. Phase 17b.3a: prefix tolerance + simple
19/// escape table. 17b.3b adds octal / hex / UCN numeric escapes in the
20/// same kernel.
21pub mod gpu_char_constant_scan;
22#[cfg(test)]
23mod gpu_char_constant_scan_tests;
24/// GPU comment-strip mask. Phase 17b.5: per-byte mask `1=comment,
25/// 0=code` covering `//` line comments and `/*…*/` block comments.
26/// Composes with `gpu_line_splice_classify` via mask-AND for the
27/// pre-lex byte filter.
28pub mod gpu_comment_strip_mask;
29#[cfg(test)]
30mod gpu_conditional_value_tests;
31/// GPU `#define` row parser. Phase 17b.6: per `TOK_PREPROC` token of
32/// kind `TOK_PP_DEFINE`, extracts macro name + optional arg-list +
33/// replacement body byte spans. Per-thread, fully parallel.
34pub mod gpu_define_parse;
35#[cfg(test)]
36mod gpu_define_parse_tests;
37/// GPU directive-metadata kernel  -  replaces the CPU
38/// `reference_c_preprocessor_directive_metadata` for production paths.
39/// Phase 17a: directive kind classification. Phase 17b will add the
40/// shunting-yard conditional evaluator in the same module.
41pub mod gpu_directive_metadata;
42mod gpu_directive_parse_shared;
43/// GPU `#if` / `#elif` expression evaluator. Phase 17b.4: per-thread
44/// iterative shunting-yard parser using fixed-depth value/operator
45/// stacks. Composes the literal scan, char-constant scan, and
46/// defined-name lookup logic. Last piece of 17b.
47pub mod gpu_if_expression;
48/// ABI helpers for the GPU `#if` / `#elif` expression evaluator.
49pub mod gpu_if_expression_abi;
50/// GPU `#ifdef` / `#ifndef` evaluator. Phase 17b.1 of the directive
51/// metadata pipeline. Composes with `gpu_directive_metadata` (which
52/// runs first to populate `directive_kinds`) and runs second to fill
53/// the `ifdef`/`ifndef` rows of `directive_values`.
54pub mod gpu_ifdef_value;
55/// GPU `#include` row parser. Phase 17b.7: per `TOK_PREPROC` token of
56/// kind `TOK_PP_INCLUDE` / `TOK_PP_INCLUDE_NEXT`, extracts the path
57/// byte span and `<…>` vs `"…"` flag. Per-thread, fully parallel.
58pub mod gpu_include_parse;
59/// GPU integer-literal scanner. Phase 17b.2: standalone scanner kernel
60/// for testing the literal-parse logic in isolation; phase 17b.4 will
61/// inline the same logic into the `#if` expression evaluator.
62pub mod gpu_int_literal_scan;
63/// GPU-resident preprocessor pipeline orchestration. Phase 18 of the
64/// Composes every kernel above into the host-side translation-unit flow.
65/// Lives beside the kernels so the unit/roundtrip tests do not have to
66/// drag in the wgpu/vyre-debug dev-dep stack.
67pub mod gpu_pipeline;
68mod gpu_source_bytes;
69/// GPU `#undef` row parser. Per `TOK_PREPROC` token of kind
70/// `TOK_PP_UNDEF`, extracts the macro-name byte span. Per-thread,
71/// fully parallel. Replaces the previous workaround of routing
72/// `#undef` rows through `gpu_define_parse` (which has a 6-byte
73/// keyword-length offset baked in for `#define`).
74pub mod gpu_undef_parse;
75/// Macro-expansion source-byte materialization helpers.
76pub mod materialization;
77/// Include source-manager ABI.
78pub mod source;
79/// Token synthesis helpers for macro stringification and token paste.
80pub mod synthesis;
81
82/// Source bytes after C translation phase 2 line splicing.
83///
84/// `bytes` contains the source with every backslash-newline pair deleted.
85/// `original_offsets` maps each output byte boundary back to the input byte
86/// boundary at the same logical position. Its length is always
87/// `bytes.len() + 1`, with the final entry pointing at `source.len()`.
88#[derive(Debug, Clone, PartialEq, Eq)]
89pub struct CLineSplicedSource {
90    /// Phase-2 source bytes with line-splice pairs removed.
91    pub bytes: Vec<u8>,
92    /// Output byte-boundary to original byte-boundary map.
93    pub original_offsets: Vec<usize>,
94}
95
96impl CLineSplicedSource {
97    /// Map a logical byte boundary in `bytes` back to an original source offset.
98    #[must_use]
99    pub fn original_offset(&self, logical_offset: usize) -> usize {
100        self.original_offsets
101            .get(logical_offset)
102            .copied()
103            .or_else(|| self.original_offsets.last().copied())
104            .unwrap_or(0)
105    }
106}
107
108/// Delete C translation phase 2 backslash-newline pairs.
109///
110/// This is intentionally global and independent of directive parsing: every C
111/// tokenization path must see the same phase-2 byte stream before directives,
112/// macro names, and ordinary tokens are interpreted.
113#[must_use]
114pub fn c_translation_phase_line_splice(source: &[u8]) -> CLineSplicedSource {
115    let mut bytes = Vec::with_capacity(source.len());
116    let mut original_offsets = Vec::with_capacity(source.len() + 1);
117    let mut index = 0usize;
118
119    while index < source.len() {
120        if source[index] == b'\\' {
121            match source.get(index + 1).copied() {
122                Some(b'\n') => {
123                    index += 2;
124                    continue;
125                }
126                Some(b'\r') => {
127                    index += 2;
128                    if source.get(index).copied() == Some(b'\n') {
129                        index += 1;
130                    }
131                    continue;
132                }
133                _ => {}
134            }
135        }
136
137        original_offsets.push(index);
138        bytes.push(source[index]);
139        index += 1;
140    }
141
142    original_offsets.push(source.len());
143    CLineSplicedSource {
144        bytes,
145        original_offsets,
146    }
147}
148
149/// Stable directive kind identifiers carried by host-side preprocessor analysis.
150#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
151pub enum CPreprocessorDirectiveKind {
152    /// Empty `#` directive.
153    Null,
154    /// `#define`.
155    Define,
156    /// `#undef`.
157    Undef,
158    /// `#include`.
159    Include,
160    /// GNU `#include_next`.
161    IncludeNext,
162    /// `#if`.
163    If,
164    /// `#ifdef`.
165    Ifdef,
166    /// `#ifndef`.
167    Ifndef,
168    /// `#elif`.
169    Elif,
170    /// `#else`.
171    Else,
172    /// `#endif`.
173    Endif,
174    /// `#pragma`.
175    Pragma,
176    /// `#line`.
177    Line,
178    /// `#error`.
179    Error,
180    /// GNU `#warning`.
181    Warning,
182    /// System `#ident`.
183    Ident,
184    /// System `#sccs`.
185    Sccs,
186    /// `#embed` (C23): bring file contents into the TU as initializers.
187    Embed,
188    /// `#elifdef` (C23): shorthand for `#elif defined(...)`.
189    Elifdef,
190    /// `#elifndef` (C23): shorthand for `#elif !defined(...)`.
191    Elifndef,
192    /// `#import` (clang/Objective-C): include-once form.
193    Import,
194}
195
196impl CPreprocessorDirectiveKind {
197    /// Return the stable directive metadata token ID.
198    #[must_use]
199    pub const fn token_id(self) -> u32 {
200        match self {
201            Self::Null => TOK_PP_NULL,
202            Self::Define => TOK_PP_DEFINE,
203            Self::Undef => TOK_PP_UNDEF,
204            Self::Include => TOK_PP_INCLUDE,
205            Self::IncludeNext => TOK_PP_INCLUDE_NEXT,
206            Self::If => TOK_PP_IF,
207            Self::Ifdef => TOK_PP_IFDEF,
208            Self::Ifndef => TOK_PP_IFNDEF,
209            Self::Elif => TOK_PP_ELIF,
210            Self::Else => TOK_PP_ELSE,
211            Self::Endif => TOK_PP_ENDIF,
212            Self::Pragma => TOK_PP_PRAGMA,
213            Self::Line => TOK_PP_LINE,
214            Self::Error => TOK_PP_ERROR,
215            Self::Warning => TOK_PP_WARNING,
216            Self::Ident => TOK_PP_IDENT,
217            Self::Sccs => TOK_PP_SCCS,
218            Self::Embed => TOK_PP_EMBED,
219            Self::Elifdef => TOK_PP_ELIFDEF,
220            Self::Elifndef => TOK_PP_ELIFNDEF,
221            Self::Import => TOK_PP_IMPORT,
222        }
223    }
224}
225
226/// Parsed metadata for one compact `TOK_PREPROC` source row.
227#[derive(Debug, Clone, Copy, PartialEq, Eq)]
228pub struct CPreprocessorDirective {
229    /// Recognized directive kind.
230    pub kind: CPreprocessorDirectiveKind,
231    /// Byte offset of the directive keyword within the phase-2 logical row.
232    pub keyword_start: usize,
233    /// Byte length of the directive keyword. Null directives use zero.
234    pub keyword_len: usize,
235    /// Byte offset where directive payload starts after horizontal whitespace.
236    pub payload_start: usize,
237    /// Byte offset where the phase-2 logical directive row ends.
238    pub logical_end: usize,
239}
240
241/// Fail-loud preprocessor row classification error.
242#[derive(Debug, Clone, PartialEq, Eq)]
243pub struct CPreprocessorError {
244    /// Byte offset where classification failed.
245    pub offset: usize,
246    /// Actionable diagnostic.
247    pub message: &'static str,
248}
249
250impl core::fmt::Display for CPreprocessorError {
251    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
252        write!(f, "{} at byte {}", self.message, self.offset)
253    }
254}
255
256impl std::error::Error for CPreprocessorError {}
257
258pub(crate) fn c_directive_payload<'a>(
259    row: &'a [u8],
260    directive: CPreprocessorDirective,
261) -> Result<&'a [u8], CPreprocessorError> {
262    row.get(directive.payload_start..directive.logical_end)
263        .ok_or(CPreprocessorError {
264            offset: directive.payload_start.min(row.len()),
265            message: "preprocessor directive payload span is outside the logical row. Fix: pass phase-2 directive spans from the same row bytes.",
266        })
267}
268
269/// Return the physical byte length of one logical preprocessing directive row.
270///
271/// C translation phase 2 deletes backslash-newline pairs before directive
272/// parsing. The returned span therefore continues across `\\\n` and `\\\r\n`
273/// pairs and stops before the first non-spliced line terminator.
274#[must_use]
275pub fn c_logical_directive_len(source: &[u8], offset: usize) -> usize {
276    if offset >= source.len() {
277        return 0;
278    }
279
280    let mut index = offset;
281    while index < source.len() {
282        match source[index] {
283            b'\n' => {
284                if index > offset && source[index - 1] == b'\\' {
285                    index += 1;
286                    continue;
287                }
288                break;
289            }
290            b'\r' => {
291                let has_lf = source.get(index + 1).copied() == Some(b'\n');
292                if index > offset && source[index - 1] == b'\\' {
293                    index += usize::from(has_lf) + 1;
294                    continue;
295                }
296                break;
297            }
298            _ => index += 1,
299        }
300    }
301
302    index - offset
303}
304
305/// Classify a compact preprocessor row without expanding macros.
306///
307/// This function validates the directive name, treats horizontal whitespace
308/// after `#` the same way C does, and leaves payload bytes untouched so macro
309/// definitions, includes, pragmas, and `#error` diagnostics share one phase-2
310/// view with downstream directive and macro handling.
311///
312/// # Errors
313///
314/// Returns a diagnostic when the row is not a directive row or uses an
315/// unsupported directive spelling.
316pub fn try_classify_preprocessor_directive(
317    row: &[u8],
318) -> Result<CPreprocessorDirective, CPreprocessorError> {
319    let logical_end = c_logical_directive_len(row, 0);
320    let physical_line = row.get(..logical_end).unwrap_or(row);
321    let spliced = c_translation_phase_line_splice(physical_line);
322    classify_phase2_preprocessor_directive(&spliced.bytes).map_err(|mut err| {
323        err.offset = spliced.original_offset(err.offset);
324        err
325    })
326}
327
328fn classify_phase2_preprocessor_directive(
329    line: &[u8],
330) -> Result<CPreprocessorDirective, CPreprocessorError> {
331    let mut index = skip_ws_and_comments(line, 0);
332    if line.get(index).copied() != Some(b'#') {
333        return Err(CPreprocessorError {
334            offset: index,
335            message: "Fix: preprocessor row must begin with # after horizontal whitespace",
336        });
337    }
338
339    index += 1;
340    index = skip_ws_and_comments(line, index);
341    if index >= line.len() {
342        return Ok(CPreprocessorDirective {
343            kind: CPreprocessorDirectiveKind::Null,
344            keyword_start: index,
345            keyword_len: 0,
346            payload_start: index,
347            logical_end: line.len(),
348        });
349    }
350
351    let keyword_start = index;
352    while index < line.len() && is_directive_ident_continue(line[index]) {
353        index += 1;
354    }
355    let keyword = &line[keyword_start..index];
356    let kind = match keyword {
357        b"define" => CPreprocessorDirectiveKind::Define,
358        b"undef" => CPreprocessorDirectiveKind::Undef,
359        b"include" => CPreprocessorDirectiveKind::Include,
360        b"include_next" => CPreprocessorDirectiveKind::IncludeNext,
361        b"if" => CPreprocessorDirectiveKind::If,
362        b"ifdef" => CPreprocessorDirectiveKind::Ifdef,
363        b"ifndef" => CPreprocessorDirectiveKind::Ifndef,
364        b"elif" => CPreprocessorDirectiveKind::Elif,
365        b"else" => CPreprocessorDirectiveKind::Else,
366        b"endif" => CPreprocessorDirectiveKind::Endif,
367        b"pragma" => CPreprocessorDirectiveKind::Pragma,
368        b"line" => CPreprocessorDirectiveKind::Line,
369        b"error" => CPreprocessorDirectiveKind::Error,
370        b"warning" => CPreprocessorDirectiveKind::Warning,
371        b"ident" => CPreprocessorDirectiveKind::Ident,
372        b"sccs" => CPreprocessorDirectiveKind::Sccs,
373        b"embed" => CPreprocessorDirectiveKind::Embed,
374        b"elifdef" => CPreprocessorDirectiveKind::Elifdef,
375        b"elifndef" => CPreprocessorDirectiveKind::Elifndef,
376        b"import" => CPreprocessorDirectiveKind::Import,
377        _ => {
378            return Err(CPreprocessorError {
379                offset: keyword_start,
380                message: "Fix: implement or reject this C preprocessor directive explicitly",
381            });
382        }
383    };
384
385    Ok(CPreprocessorDirective {
386        kind,
387        keyword_start,
388        keyword_len: keyword.len(),
389        payload_start: skip_ws_and_comments(line, index),
390        logical_end: line.len(),
391    })
392}
393
394/// Build directive-kind and conditional-value metadata for compact C tokens.
395///
396/// `TOK_PREPROC` rows are classified from original source spans. Conditional
397/// rows get an evaluated truth value; all other rows get `0`.
398///
399/// # Errors
400///
401/// Returns a diagnostic when token streams are inconsistent, a directive span
402/// is outside `source`, or the current payload evaluator cannot parse a
403/// conditional expression.
404#[cfg(any(test, feature = "cpu-parity"))]
405pub fn reference_c_preprocessor_directive_metadata(
406    tok_types: &[u32],
407    tok_starts: &[u32],
408    tok_lens: &[u32],
409    source: &[u8],
410    defined_macros: &[&[u8]],
411) -> Result<(Vec<u32>, Vec<u32>), CPreprocessorError> {
412    let mut directive_kinds = vec![0; tok_types.len()];
413    let mut directive_values = vec![0; tok_types.len()];
414    directive_scan::for_each_directive_row(tok_types, tok_starts, tok_lens, source, |row| {
415        let scan = ScannedDirective::classify(row.bytes, row.start)?;
416        directive_kinds[row.index] = scan.directive.kind.token_id();
417        directive_values[row.index] =
418            conditional_directive_value(&scan.spliced.bytes, scan.directive, defined_macros)
419                .map_err(|err| scan.remap(err))?
420                .unwrap_or(0);
421        Ok(())
422    })?;
423    Ok((directive_kinds, directive_values))
424}
425
426fn conditional_directive_value(
427    row: &[u8],
428    directive: CPreprocessorDirective,
429    defined_macros: &[&[u8]],
430) -> Result<Option<u32>, CPreprocessorError> {
431    let payload = c_directive_payload(row, directive)?;
432    match directive.kind {
433        CPreprocessorDirectiveKind::If | CPreprocessorDirectiveKind::Elif => Ok(Some(u32::from(
434            PreprocessorExprParser {
435                bytes: payload,
436                index: 0,
437                base_offset: directive.payload_start,
438                defined_macros,
439                depth: 0,
440            }
441            .parse()?,
442        ))),
443        CPreprocessorDirectiveKind::Ifdef => Ok(Some(u32::from(
444            first_payload_ident(payload).is_some_and(|name| macro_is_defined(defined_macros, name)),
445        ))),
446        CPreprocessorDirectiveKind::Ifndef => Ok(Some(u32::from(
447            first_payload_ident(payload)
448                .is_some_and(|name| !macro_is_defined(defined_macros, name)),
449        ))),
450        _ => Ok(None),
451    }
452}
453
454mod expr_parser;
455pub use expr_parser::is_reserved_preprocessor_identifier;
456use expr_parser::PreprocessorExprParser;
457
458pub(super) fn first_payload_ident(payload: &[u8]) -> Option<&[u8]> {
459    let mut index = skip_ws_and_comments(payload, 0);
460    let start = index;
461    if !payload.get(index).copied().is_some_and(is_c_ident_start) {
462        return None;
463    }
464    index += 1;
465    while payload
466        .get(index)
467        .copied()
468        .is_some_and(is_directive_ident_continue)
469    {
470        index += 1;
471    }
472    payload.get(start..index)
473}
474
475#[inline]
476pub(super) fn macro_is_defined(defined_macros: &[&[u8]], name: &[u8]) -> bool {
477    defined_macros.iter().any(|candidate| *candidate == name)
478}
479
480/// Skip horizontal whitespace and comments in a directive line.
481///
482/// Directive classification runs before the comment-strip stage, so a comment
483/// here is still literal bytes and counts as whitespace. Payload scanning runs
484/// after that stage and uses [`directive_scan::skip_horizontal_ws`] instead.
485#[inline]
486pub(super) fn skip_ws_and_comments(bytes: &[u8], mut index: usize) -> usize {
487    loop {
488        match bytes.get(index).copied() {
489            Some(b' ' | b'\t' | b'\x0b' | b'\x0c') => index += 1,
490            Some(b'/') if bytes.get(index + 1).copied() == Some(b'/') => {
491                return bytes.len();
492            }
493            Some(b'/') if bytes.get(index + 1).copied() == Some(b'*') => {
494                index += 2;
495                while index + 1 < bytes.len() && bytes.get(index..index + 2) != Some(b"*/") {
496                    index += 1;
497                }
498                if index + 1 >= bytes.len() {
499                    return bytes.len();
500                }
501                index += 2;
502            }
503            _ => return index,
504        }
505    }
506}
507
508#[inline]
509pub(super) fn is_directive_ident_continue(byte: u8) -> bool {
510    byte.is_ascii_alphanumeric() || byte == b'_'
511}
512
513#[inline]
514pub(super) fn is_c_ident_start(byte: u8) -> bool {
515    byte.is_ascii_alphabetic() || byte == b'_'
516}
517
518#[cfg(test)]
519mod tests {
520    use super::{c_directive_payload, CPreprocessorDirective, CPreprocessorDirectiveKind};
521
522    #[test]
523    fn directive_payload_rejects_corrupt_span_instead_of_defaulting_empty() {
524        let directive = CPreprocessorDirective {
525            kind: CPreprocessorDirectiveKind::If,
526            keyword_start: 1,
527            keyword_len: 2,
528            payload_start: 8,
529            logical_end: 4,
530        };
531        let err = c_directive_payload(b"#if 1", directive)
532            .expect_err("corrupt directive spans must fail loudly");
533        assert_eq!(err.offset, 5);
534        assert!(
535            err.message.contains("payload span is outside"),
536            "error must explain the corrupted payload span"
537        );
538    }
539}
vyre_libs/parsing/c/preprocess/mod.rs

vyre_libs/parsing/c/preprocess/
mod.rs
