vyre_libs/parsing/c/preprocess/
mod.rs

1//! C11 preprocessor passes.
2
3#[cfg(any(test, feature = "cpu-parity"))]
4use crate::parsing::c::lex::tokens::TOK_PREPROC;
5use crate::parsing::c::lex::tokens::{
6    TOK_PP_DEFINE, TOK_PP_ELIF, TOK_PP_ELIFDEF, TOK_PP_ELIFNDEF, TOK_PP_ELSE, TOK_PP_EMBED,
7    TOK_PP_ENDIF, TOK_PP_ERROR, TOK_PP_IDENT, TOK_PP_IF, TOK_PP_IFDEF, TOK_PP_IFNDEF,
8    TOK_PP_IMPORT, TOK_PP_INCLUDE, TOK_PP_INCLUDE_NEXT, TOK_PP_LINE, TOK_PP_NULL, TOK_PP_PRAGMA,
9    TOK_PP_SCCS, TOK_PP_UNDEF, TOK_PP_WARNING,
10};
11
12/// Preprocessor side-effect metadata.
13pub mod effects;
14/// Macro-expansion kernel.
15pub mod expansion;
16/// GPU char-constant scanner. Phase 17b.3a: prefix tolerance + simple
17/// escape table. 17b.3b adds octal / hex / UCN numeric escapes in the
18/// same kernel.
19pub mod gpu_char_constant_scan;
20#[cfg(test)]
21mod gpu_char_constant_scan_tests;
22/// GPU comment-strip mask. Phase 17b.5: per-byte mask `1=comment,
23/// 0=code` covering `//` line comments and `/*…*/` block comments.
24/// Composes with `gpu_line_splice_classify` via mask-AND for the
25/// pre-lex byte filter.
26pub mod gpu_comment_strip_mask;
27#[cfg(test)]
28mod gpu_comment_strip_mask_tests;
29#[cfg(test)]
30mod gpu_conditional_value_tests;
31/// GPU `#define` row parser. Phase 17b.6: per `TOK_PREPROC` token of
32/// kind `TOK_PP_DEFINE`, extracts macro name + optional arg-list +
33/// replacement body byte spans. Per-thread, fully parallel.
34pub mod gpu_define_parse;
35#[cfg(test)]
36mod gpu_define_parse_tests;
37/// GPU directive-metadata kernel  -  replaces the CPU
38/// `reference_c_preprocessor_directive_metadata` for production paths.
39/// Phase 17a: directive kind classification. Phase 17b will add the
40/// shunting-yard conditional evaluator in the same module.
41pub mod gpu_directive_metadata;
42mod gpu_directive_parse_shared;
43/// GPU `#if` / `#elif` expression evaluator. Phase 17b.4: per-thread
44/// iterative shunting-yard parser using fixed-depth value/operator
45/// stacks. Composes the literal scan, char-constant scan, and
46/// defined-name lookup logic. Last piece of 17b.
47pub mod gpu_if_expression;
48/// ABI helpers for the GPU `#if` / `#elif` expression evaluator.
49pub mod gpu_if_expression_abi;
50/// GPU `#ifdef` / `#ifndef` evaluator. Phase 17b.1 of the directive
51/// metadata pipeline. Composes with `gpu_directive_metadata` (which
52/// runs first to populate `directive_kinds`) and runs second to fill
53/// the `ifdef`/`ifndef` rows of `directive_values`.
54pub mod gpu_ifdef_value;
55/// GPU `#include` row parser. Phase 17b.7: per `TOK_PREPROC` token of
56/// kind `TOK_PP_INCLUDE` / `TOK_PP_INCLUDE_NEXT`, extracts the path
57/// byte span and `<…>` vs `"…"` flag. Per-thread, fully parallel.
58pub mod gpu_include_parse;
59/// GPU integer-literal scanner. Phase 17b.2: standalone scanner kernel
60/// for testing the literal-parse logic in isolation; phase 17b.4 will
61/// inline the same logic into the `#if` expression evaluator.
62pub mod gpu_int_literal_scan;
63/// GPU-resident preprocessor pipeline orchestration. Phase 18 of the
64/// v0.4 plan: composes every kernel above into the host-side flow that
65/// `vyre-frontend-c::tu_host` calls. Lives here (not in
66/// vyre-frontend-c) so the unit/roundtrip tests don't have to drag in
67/// the wgpu/vyre-debug dev-dep stack.
68pub mod gpu_pipeline;
69mod gpu_source_bytes;
70/// GPU `#undef` row parser. Per `TOK_PREPROC` token of kind
71/// `TOK_PP_UNDEF`, extracts the macro-name byte span. Per-thread,
72/// fully parallel. Replaces the previous workaround of routing
73/// `#undef` rows through `gpu_define_parse` (which has a 6-byte
74/// keyword-length offset baked in for `#define`).
75pub mod gpu_undef_parse;
76/// Macro-expansion source-byte materialization helpers.
77pub mod materialization;
78/// Include source-manager ABI.
79pub mod source;
80/// Token synthesis helpers for macro stringification and token paste.
81pub mod synthesis;
82
83/// Source bytes after C translation phase 2 line splicing.
84///
85/// `bytes` contains the source with every backslash-newline pair deleted.
86/// `original_offsets` maps each output byte boundary back to the input byte
87/// boundary at the same logical position. Its length is always
88/// `bytes.len() + 1`, with the final entry pointing at `source.len()`.
89#[derive(Debug, Clone, PartialEq, Eq)]
90pub struct CLineSplicedSource {
91    /// Phase-2 source bytes with line-splice pairs removed.
92    pub bytes: Vec<u8>,
93    /// Output byte-boundary to original byte-boundary map.
94    pub original_offsets: Vec<usize>,
95}
96
97impl CLineSplicedSource {
98    /// Map a logical byte boundary in `bytes` back to an original source offset.
99    #[must_use]
100    pub fn original_offset(&self, logical_offset: usize) -> usize {
101        self.original_offsets
102            .get(logical_offset)
103            .copied()
104            .or_else(|| self.original_offsets.last().copied())
105            .unwrap_or(0)
106    }
107}
108
109/// Delete C translation phase 2 backslash-newline pairs.
110///
111/// This is intentionally global and independent of directive parsing: every C
112/// tokenization path must see the same phase-2 byte stream before directives,
113/// macro names, and ordinary tokens are interpreted.
114#[must_use]
115pub fn c_translation_phase_line_splice(source: &[u8]) -> CLineSplicedSource {
116    let mut bytes = Vec::with_capacity(source.len());
117    let mut original_offsets = Vec::with_capacity(source.len() + 1);
118    let mut index = 0usize;
119
120    while index < source.len() {
121        if source[index] == b'\\' {
122            match source.get(index + 1).copied() {
123                Some(b'\n') => {
124                    index += 2;
125                    continue;
126                }
127                Some(b'\r') => {
128                    index += 2;
129                    if source.get(index).copied() == Some(b'\n') {
130                        index += 1;
131                    }
132                    continue;
133                }
134                _ => {}
135            }
136        }
137
138        original_offsets.push(index);
139        bytes.push(source[index]);
140        index += 1;
141    }
142
143    original_offsets.push(source.len());
144    CLineSplicedSource {
145        bytes,
146        original_offsets,
147    }
148}
149
150/// Stable directive kind identifiers carried by host-side preprocessor analysis.
151#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
152pub enum CPreprocessorDirectiveKind {
153    /// Empty `#` directive.
154    Null,
155    /// `#define`.
156    Define,
157    /// `#undef`.
158    Undef,
159    /// `#include`.
160    Include,
161    /// GNU `#include_next`.
162    IncludeNext,
163    /// `#if`.
164    If,
165    /// `#ifdef`.
166    Ifdef,
167    /// `#ifndef`.
168    Ifndef,
169    /// `#elif`.
170    Elif,
171    /// `#else`.
172    Else,
173    /// `#endif`.
174    Endif,
175    /// `#pragma`.
176    Pragma,
177    /// `#line`.
178    Line,
179    /// `#error`.
180    Error,
181    /// GNU `#warning`.
182    Warning,
183    /// System `#ident`.
184    Ident,
185    /// System `#sccs`.
186    Sccs,
187    /// `#embed` (C23): bring file contents into the TU as initializers.
188    Embed,
189    /// `#elifdef` (C23): shorthand for `#elif defined(...)`.
190    Elifdef,
191    /// `#elifndef` (C23): shorthand for `#elif !defined(...)`.
192    Elifndef,
193    /// `#import` (clang/Objective-C): include-once form.
194    Import,
195}
196
197impl CPreprocessorDirectiveKind {
198    /// Return the stable directive metadata token ID.
199    #[must_use]
200    pub const fn token_id(self) -> u32 {
201        match self {
202            Self::Null => TOK_PP_NULL,
203            Self::Define => TOK_PP_DEFINE,
204            Self::Undef => TOK_PP_UNDEF,
205            Self::Include => TOK_PP_INCLUDE,
206            Self::IncludeNext => TOK_PP_INCLUDE_NEXT,
207            Self::If => TOK_PP_IF,
208            Self::Ifdef => TOK_PP_IFDEF,
209            Self::Ifndef => TOK_PP_IFNDEF,
210            Self::Elif => TOK_PP_ELIF,
211            Self::Else => TOK_PP_ELSE,
212            Self::Endif => TOK_PP_ENDIF,
213            Self::Pragma => TOK_PP_PRAGMA,
214            Self::Line => TOK_PP_LINE,
215            Self::Error => TOK_PP_ERROR,
216            Self::Warning => TOK_PP_WARNING,
217            Self::Ident => TOK_PP_IDENT,
218            Self::Sccs => TOK_PP_SCCS,
219            Self::Embed => TOK_PP_EMBED,
220            Self::Elifdef => TOK_PP_ELIFDEF,
221            Self::Elifndef => TOK_PP_ELIFNDEF,
222            Self::Import => TOK_PP_IMPORT,
223        }
224    }
225}
226
227/// Parsed metadata for one compact `TOK_PREPROC` source row.
228#[derive(Debug, Clone, Copy, PartialEq, Eq)]
229pub struct CPreprocessorDirective {
230    /// Recognized directive kind.
231    pub kind: CPreprocessorDirectiveKind,
232    /// Byte offset of the directive keyword within the phase-2 logical row.
233    pub keyword_start: usize,
234    /// Byte length of the directive keyword. Null directives use zero.
235    pub keyword_len: usize,
236    /// Byte offset where directive payload starts after horizontal whitespace.
237    pub payload_start: usize,
238    /// Byte offset where the phase-2 logical directive row ends.
239    pub logical_end: usize,
240}
241
242/// Fail-loud preprocessor row classification error.
243#[derive(Debug, Clone, PartialEq, Eq)]
244pub struct CPreprocessorError {
245    /// Byte offset where classification failed.
246    pub offset: usize,
247    /// Actionable diagnostic.
248    pub message: &'static str,
249}
250
251impl core::fmt::Display for CPreprocessorError {
252    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
253        write!(f, "{} at byte {}", self.message, self.offset)
254    }
255}
256
257impl std::error::Error for CPreprocessorError {}
258
259pub(crate) fn c_directive_payload<'a>(
260    row: &'a [u8],
261    directive: CPreprocessorDirective,
262) -> Result<&'a [u8], CPreprocessorError> {
263    row.get(directive.payload_start..directive.logical_end)
264        .ok_or(CPreprocessorError {
265            offset: directive.payload_start.min(row.len()),
266            message: "preprocessor directive payload span is outside the logical row. Fix: pass phase-2 directive spans from the same row bytes.",
267        })
268}
269
270/// Return the physical byte length of one logical preprocessing directive row.
271///
272/// C translation phase 2 deletes backslash-newline pairs before directive
273/// parsing. The returned span therefore continues across `\\\n` and `\\\r\n`
274/// pairs and stops before the first non-spliced line terminator.
275#[must_use]
276pub fn c_logical_directive_len(source: &[u8], offset: usize) -> usize {
277    if offset >= source.len() {
278        return 0;
279    }
280
281    let mut index = offset;
282    while index < source.len() {
283        match source[index] {
284            b'\n' => {
285                if index > offset && source[index - 1] == b'\\' {
286                    index += 1;
287                    continue;
288                }
289                break;
290            }
291            b'\r' => {
292                let has_lf = source.get(index + 1).copied() == Some(b'\n');
293                if index > offset && source[index - 1] == b'\\' {
294                    index += usize::from(has_lf) + 1;
295                    continue;
296                }
297                break;
298            }
299            _ => index += 1,
300        }
301    }
302
303    index - offset
304}
305
306/// Classify a compact preprocessor row without expanding macros.
307///
308/// This function validates the directive name, treats horizontal whitespace
309/// after `#` the same way C does, and leaves payload bytes untouched so macro
310/// definitions, includes, pragmas, and `#error` diagnostics share one phase-2
311/// view with downstream directive and macro handling.
312///
313/// # Errors
314///
315/// Returns a diagnostic when the row is not a directive row or uses an
316/// unsupported directive spelling.
317pub fn try_classify_preprocessor_directive(
318    row: &[u8],
319) -> Result<CPreprocessorDirective, CPreprocessorError> {
320    let logical_end = c_logical_directive_len(row, 0);
321    let physical_line = row.get(..logical_end).unwrap_or(row);
322    let spliced = c_translation_phase_line_splice(physical_line);
323    classify_phase2_preprocessor_directive(&spliced.bytes).map_err(|mut err| {
324        err.offset = spliced.original_offset(err.offset);
325        err
326    })
327}
328
329fn classify_phase2_preprocessor_directive(
330    line: &[u8],
331) -> Result<CPreprocessorDirective, CPreprocessorError> {
332    let mut index = skip_horizontal_ws(line, 0);
333    if line.get(index).copied() != Some(b'#') {
334        return Err(CPreprocessorError {
335            offset: index,
336            message: "Fix: preprocessor row must begin with # after horizontal whitespace",
337        });
338    }
339
340    index += 1;
341    index = skip_horizontal_ws(line, index);
342    if index >= line.len() {
343        return Ok(CPreprocessorDirective {
344            kind: CPreprocessorDirectiveKind::Null,
345            keyword_start: index,
346            keyword_len: 0,
347            payload_start: index,
348            logical_end: line.len(),
349        });
350    }
351
352    let keyword_start = index;
353    while index < line.len() && is_directive_ident_continue(line[index]) {
354        index += 1;
355    }
356    let keyword = &line[keyword_start..index];
357    let kind = match keyword {
358        b"define" => CPreprocessorDirectiveKind::Define,
359        b"undef" => CPreprocessorDirectiveKind::Undef,
360        b"include" => CPreprocessorDirectiveKind::Include,
361        b"include_next" => CPreprocessorDirectiveKind::IncludeNext,
362        b"if" => CPreprocessorDirectiveKind::If,
363        b"ifdef" => CPreprocessorDirectiveKind::Ifdef,
364        b"ifndef" => CPreprocessorDirectiveKind::Ifndef,
365        b"elif" => CPreprocessorDirectiveKind::Elif,
366        b"else" => CPreprocessorDirectiveKind::Else,
367        b"endif" => CPreprocessorDirectiveKind::Endif,
368        b"pragma" => CPreprocessorDirectiveKind::Pragma,
369        b"line" => CPreprocessorDirectiveKind::Line,
370        b"error" => CPreprocessorDirectiveKind::Error,
371        b"warning" => CPreprocessorDirectiveKind::Warning,
372        b"ident" => CPreprocessorDirectiveKind::Ident,
373        b"sccs" => CPreprocessorDirectiveKind::Sccs,
374        b"embed" => CPreprocessorDirectiveKind::Embed,
375        b"elifdef" => CPreprocessorDirectiveKind::Elifdef,
376        b"elifndef" => CPreprocessorDirectiveKind::Elifndef,
377        b"import" => CPreprocessorDirectiveKind::Import,
378        _ => {
379            return Err(CPreprocessorError {
380                offset: keyword_start,
381                message: "Fix: implement or reject this C preprocessor directive explicitly",
382            });
383        }
384    };
385
386    Ok(CPreprocessorDirective {
387        kind,
388        keyword_start,
389        keyword_len: keyword.len(),
390        payload_start: skip_horizontal_ws(line, index),
391        logical_end: line.len(),
392    })
393}
394
395/// Build directive-kind and conditional-value metadata for compact C tokens.
396///
397/// `TOK_PREPROC` rows are classified from original source spans. Conditional
398/// rows get an evaluated truth value; all other rows get `0`.
399///
400/// # Errors
401///
402/// Returns a diagnostic when token streams are inconsistent, a directive span
403/// is outside `source`, or the current payload evaluator cannot parse a
404/// conditional expression.
405#[deprecated(
406    note = "CPU reference oracle only; production C preprocessing must use the GPU directive metadata pipeline"
407)]
408#[cfg(any(test, feature = "cpu-parity"))]
409pub fn reference_c_preprocessor_directive_metadata(
410    tok_types: &[u32],
411    tok_starts: &[u32],
412    tok_lens: &[u32],
413    source: &[u8],
414    defined_macros: &[&[u8]],
415) -> Result<(Vec<u32>, Vec<u32>), CPreprocessorError> {
416    if tok_types.len() != tok_starts.len() || tok_types.len() != tok_lens.len() {
417        return Err(CPreprocessorError {
418            offset: tok_types.len().min(tok_starts.len()).min(tok_lens.len()),
419            message: "Fix: token type/start/length streams must have identical lengths",
420        });
421    }
422
423    let mut directive_kinds = vec![0; tok_types.len()];
424    let mut directive_values = vec![0; tok_types.len()];
425    for (idx, ((tok_type, start), len)) in
426        tok_types.iter().zip(tok_starts).zip(tok_lens).enumerate()
427    {
428        if *tok_type != TOK_PREPROC {
429            continue;
430        }
431        let start = usize::try_from(*start).map_err(|_| CPreprocessorError {
432            offset: idx,
433            message: "Fix: token start does not fit host usize",
434        })?;
435        let len = usize::try_from(*len).map_err(|_| CPreprocessorError {
436            offset: idx,
437            message: "Fix: token length does not fit host usize",
438        })?;
439        let token_end = start.checked_add(len).ok_or(CPreprocessorError {
440            offset: start,
441            message: "Fix: token span overflows source address space",
442        })?;
443        let physical_logical_len = c_logical_directive_len(source, start);
444        if physical_logical_len > len {
445            return Err(CPreprocessorError {
446                offset: start + len,
447                message:
448                    "Fix: TOK_PREPROC span must include the full phase-2 spliced directive row",
449            });
450        }
451        let logical_end = start
452            .checked_add(physical_logical_len)
453            .ok_or(CPreprocessorError {
454                offset: start,
455                message: "Fix: directive logical span overflows source address space",
456            })?;
457        if token_end > source.len() {
458            return Err(CPreprocessorError {
459                offset: start,
460                message: "Fix: preprocessor token span must be inside the source buffer",
461            });
462        }
463        let row = source.get(start..logical_end).ok_or(CPreprocessorError {
464            offset: start,
465            message: "Fix: preprocessor token span must be inside the source buffer",
466        })?;
467        let spliced = c_translation_phase_line_splice(row);
468        let directive =
469            classify_phase2_preprocessor_directive(&spliced.bytes).map_err(|mut err| {
470                err.offset = start + spliced.original_offset(err.offset);
471                err
472            })?;
473        directive_kinds[idx] = directive.kind.token_id();
474        directive_values[idx] =
475            conditional_directive_value(&spliced.bytes, directive, defined_macros)
476                .map_err(|mut err| {
477                    err.offset = start + spliced.original_offset(err.offset);
478                    err
479                })?
480                .unwrap_or(0);
481    }
482    Ok((directive_kinds, directive_values))
483}
484
485fn conditional_directive_value(
486    row: &[u8],
487    directive: CPreprocessorDirective,
488    defined_macros: &[&[u8]],
489) -> Result<Option<u32>, CPreprocessorError> {
490    let payload = c_directive_payload(row, directive)?;
491    match directive.kind {
492        CPreprocessorDirectiveKind::If | CPreprocessorDirectiveKind::Elif => Ok(Some(u32::from(
493            PreprocessorExprParser {
494                bytes: payload,
495                index: 0,
496                base_offset: directive.payload_start,
497                defined_macros,
498                depth: 0,
499            }
500            .parse()?,
501        ))),
502        CPreprocessorDirectiveKind::Ifdef => Ok(Some(u32::from(
503            first_payload_ident(payload).is_some_and(|name| macro_is_defined(defined_macros, name)),
504        ))),
505        CPreprocessorDirectiveKind::Ifndef => Ok(Some(u32::from(
506            first_payload_ident(payload)
507                .is_some_and(|name| !macro_is_defined(defined_macros, name)),
508        ))),
509        _ => Ok(None),
510    }
511}
512
513mod expr_parser;
514pub use expr_parser::is_reserved_preprocessor_identifier;
515use expr_parser::PreprocessorExprParser;
516
517pub(super) fn first_payload_ident(payload: &[u8]) -> Option<&[u8]> {
518    let mut index = skip_horizontal_ws(payload, 0);
519    let start = index;
520    if !payload.get(index).copied().is_some_and(is_c_ident_start) {
521        return None;
522    }
523    index += 1;
524    while payload
525        .get(index)
526        .copied()
527        .is_some_and(is_directive_ident_continue)
528    {
529        index += 1;
530    }
531    payload.get(start..index)
532}
533
534#[inline]
535pub(super) fn macro_is_defined(defined_macros: &[&[u8]], name: &[u8]) -> bool {
536    defined_macros.iter().any(|candidate| *candidate == name)
537}
538
539#[inline]
540pub(super) fn skip_horizontal_ws(bytes: &[u8], mut index: usize) -> usize {
541    loop {
542        match bytes.get(index).copied() {
543            Some(b' ' | b'\t' | b'\x0b' | b'\x0c') => index += 1,
544            Some(b'/') if bytes.get(index + 1).copied() == Some(b'/') => {
545                return bytes.len();
546            }
547            Some(b'/') if bytes.get(index + 1).copied() == Some(b'*') => {
548                index += 2;
549                while index + 1 < bytes.len() && bytes.get(index..index + 2) != Some(b"*/") {
550                    index += 1;
551                }
552                if index + 1 >= bytes.len() {
553                    return bytes.len();
554                }
555                index += 2;
556            }
557            _ => return index,
558        }
559    }
560}
561
562#[inline]
563pub(super) fn is_directive_ident_continue(byte: u8) -> bool {
564    byte.is_ascii_alphanumeric() || byte == b'_'
565}
566
567#[inline]
568pub(super) fn is_c_ident_start(byte: u8) -> bool {
569    byte.is_ascii_alphabetic() || byte == b'_'
570}
571
572#[cfg(test)]
573mod tests {
574    use super::{c_directive_payload, CPreprocessorDirective, CPreprocessorDirectiveKind};
575
576    #[test]
577    fn directive_payload_rejects_corrupt_span_instead_of_defaulting_empty() {
578        let directive = CPreprocessorDirective {
579            kind: CPreprocessorDirectiveKind::If,
580            keyword_start: 1,
581            keyword_len: 2,
582            payload_start: 8,
583            logical_end: 4,
584        };
585        let err = c_directive_payload(b"#if 1", directive)
586            .expect_err("corrupt directive spans must fail loudly");
587        assert_eq!(err.offset, 5);
588        assert!(
589            err.message.contains("payload span is outside"),
590            "error must explain the corrupted payload span"
591        );
592    }
593}
vyre_libs/parsing/c/preprocess/mod.rs

vyre_libs/parsing/c/preprocess/
mod.rs
