cpp_code_analysis 0.2.0

The analysis layer of cpp_ls: the preprocessor, symbol scopes, type inference and the semantic checks.
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
//! Preprocessor directives, structured.
//!
//! A directive is a **line**, not a construct: it starts at a `#` that is the first token of its logical
//! line and runs to the newline that ends it. That is why this module reads the token stream rather than
//! the tree — the tree's `PreprocessorDirective` nodes are one view of the same thing, and the rule that
//! decides where a directive begins belongs to the line, not to the grammar.
//!
//! Every consumer of a directive then has to derive the same things: where the name ends, whether the next
//! token is a parameter list, what the condition expression is, whether `#include` used `<...>` or `"..."`.
//! This module derives them once.
//!
//! # What is *not* decided here
//!
//! Whether a condition is true. `#if` is not evaluated during this pass because evaluating it needs the
//! macro table at that point in the file, and the table is built *by* this pass — the two are one
//! traversal. So a condition is carried as its tokens, and [`crate::condition`] evaluates it against a
//! table when the caller has one.
//!
//! Nothing is reported as an error either. A directive whose arguments do not parse is
//! [`Directive::Other`], not a diagnostic: a file mid-edit has directives without arguments, and an
//! editor that flags `#if` on the line the user is still typing is worse than one that stays quiet.

use cpp_parser::{CppTokenData, CppTokenKind, SourceRange};

use crate::{
    macros::{MacroDef, parse_define},
    token::{Token, is_trivia},
};

/// The directive's name, as a typed value.
///
/// The name is a plain identifier in the token stream — `include`, `define`, `if` are not keywords —
/// which is why this is a closed enum built from spelling rather than from token kinds.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum DirectiveKind {
    /// `#if` — a constant expression.
    If,
    /// `#ifdef` — a name.
    Ifdef,
    /// `#ifndef` — a name.
    Ifndef,
    /// `#elif` — a constant expression.
    Elif,
    /// `#else`
    Else,
    /// `#endif`
    Endif,
    /// `#define`
    Define,
    /// `#undef`
    Undef,
    /// `#include`
    Include,
    /// `#include_next` — a GCC extension used inside system headers.
    IncludeNext,
    /// `#pragma`
    Pragma,
    /// `#error`
    Error,
    /// `#warning`
    Warning,
    /// `#line`
    Line,
    /// A `#` alone on a line, which is legal and does nothing.
    Null,
    /// A name this layer does not know. Not an error — implementations add their own.
    Unknown,
}

impl DirectiveKind {
    /// Read a directive name.
    pub fn from_name(name: &str) -> DirectiveKind {
        match name {
            "if" => DirectiveKind::If,
            "ifdef" => DirectiveKind::Ifdef,
            "ifndef" => DirectiveKind::Ifndef,
            "elif" => DirectiveKind::Elif,
            "else" => DirectiveKind::Else,
            "endif" => DirectiveKind::Endif,
            "define" => DirectiveKind::Define,
            "undef" => DirectiveKind::Undef,
            "include" => DirectiveKind::Include,
            "include_next" => DirectiveKind::IncludeNext,
            "pragma" => DirectiveKind::Pragma,
            "error" => DirectiveKind::Error,
            "warning" => DirectiveKind::Warning,
            "line" => DirectiveKind::Line,
            _ => DirectiveKind::Unknown,
        }
    }

    pub fn name(self) -> &'static str {
        match self {
            DirectiveKind::If => "if",
            DirectiveKind::Ifdef => "ifdef",
            DirectiveKind::Ifndef => "ifndef",
            DirectiveKind::Elif => "elif",
            DirectiveKind::Else => "else",
            DirectiveKind::Endif => "endif",
            DirectiveKind::Define => "define",
            DirectiveKind::Undef => "undef",
            DirectiveKind::Include => "include",
            DirectiveKind::IncludeNext => "include_next",
            DirectiveKind::Pragma => "pragma",
            DirectiveKind::Error => "error",
            DirectiveKind::Warning => "warning",
            DirectiveKind::Line => "line",
            DirectiveKind::Null => "#",
            DirectiveKind::Unknown => "",
        }
    }

    /// Does this directive open a conditional region?
    pub fn opens_a_condition(self) -> bool {
        matches!(
            self,
            DirectiveKind::If | DirectiveKind::Ifdef | DirectiveKind::Ifndef
        )
    }

    /// Does this directive continue or close one?
    pub fn closes_a_condition(self) -> bool {
        matches!(
            self,
            DirectiveKind::Elif | DirectiveKind::Else | DirectiveKind::Endif
        )
    }

    /// Does this directive define or undefine a macro, and so change the macro table?
    pub fn changes_macros(self) -> bool {
        matches!(self, DirectiveKind::Define | DirectiveKind::Undef)
    }
}

/// How an `#include` spelled its target.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum IncludeForm {
    /// `#include <vector>` — searched on the system path.
    Angle,
    /// `#include "local.h"` — searched relative to the including file first.
    Quote,
    /// `#include HEADER` — the target is a macro, and is not known until expansion.
    ///
    /// This is not rare in real code (`#include BOOST_VERSION_HEADER`), and treating it as a malformed
    /// include would be wrong: the directive is well formed, its target is just not a literal.
    Macro,
}

/// An `#include` or `#include_next`.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Include {
    pub form: IncludeForm,
    /// The target without its delimiters, or the macro's spelling for [`IncludeForm::Macro`].
    pub target: Box<str>,
    /// Whether the directive was `#include_next`.
    pub is_next: bool,
}

/// A `#define`.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Define {
    /// `None` when the directive is a `#define` whose arguments do not read — `#define 1 2`, or one
    /// being typed. The directive is still recognised, which is what lets a consumer see the line and
    /// a diagnostic say "this macro has no name" rather than "unknown directive".
    pub macro_def: Option<MacroDef>,
    /// A redefinition of a macro that is already defined, with a *different* body.
    ///
    /// Legal only when the two are identical, and a compiler warns when they are not — but this is the
    /// normal state of a header being edited, so it is recorded rather than reported as a parse
    /// failure.
    pub is_redefinition: bool,
}

/// One directive, with its arguments read out.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Directive {
    /// `#if` / `#elif`: the condition's tokens, unevaluated.
    ///
    /// Unparsed as well as unevaluated: turning tokens into an expression needs to know which
    /// identifiers are macros, and that needs the table. See [`crate::condition::parse_condition`].
    Conditional {
        kind: DirectiveKind,
        condition: Vec<Token>,
    },
    /// `#ifdef` / `#ifndef`: the name tested.
    Ifdef {
        kind: DirectiveKind,
        name: Box<str>,
    },
    /// `#else` / `#endif`: no arguments to read.
    Marker {
        kind: DirectiveKind,
    },
    Define(Define),
    Undef {
        /// `None` when the directive has no readable name, which is the state of `#undef` with the
        /// cursor after it.
        name: Option<Box<str>>,
    },
    Include(Include),
    /// `#pragma ...`: the tokens after the name.
    ///
    /// Not interpreted. `#pragma once` and `#pragma pack` are the two that matter and neither needs a
    /// grammar; a layer that claimed to understand every pragma would have to understand every
    /// compiler's extensions.
    Pragma {
        tokens: Vec<Token>,
    },
    /// `#error` or `#warning`, with its message as written.
    Diagnostic {
        kind: DirectiveKind,
        message: String,
    },
    /// `#line ...`, with its arguments.
    Line {
        tokens: Vec<Token>,
    },
    /// A `#` alone on a line.
    Null,
    /// A name this layer does not know, or one whose arguments did not read.
    ///
    /// Carries the name so that a consumer can still tell `#pragma_custom` from ``.
    Other {
        name: Box<str>,
    },
}

impl Directive {
    pub fn kind(&self) -> DirectiveKind {
        match self {
            Directive::Conditional { kind, .. }
            | Directive::Ifdef { kind, .. }
            | Directive::Marker { kind }
            | Directive::Diagnostic { kind, .. } => *kind,
            Directive::Define(_) => DirectiveKind::Define,
            Directive::Undef { .. } => DirectiveKind::Undef,
            Directive::Include(include) => {
                if include.is_next {
                    DirectiveKind::IncludeNext
                } else {
                    DirectiveKind::Include
                }
            }
            Directive::Pragma { .. } => DirectiveKind::Pragma,
            Directive::Line { .. } => DirectiveKind::Line,
            Directive::Null => DirectiveKind::Null,
            Directive::Other { .. } => DirectiveKind::Unknown,
        }
    }

    /// The macro this directive introduces, for the table.
    pub fn defines(&self) -> Option<&MacroDef> {
        match self {
            Directive::Define(define) => define.macro_def.as_ref(),
            _ => None,
        }
    }

    /// The name this directive undefines, if it has one.
    pub fn undefines(&self) -> Option<&str> {
        match self {
            Directive::Undef { name } => name.as_deref(),
            _ => None,
        }
    }

    /// The include this directive is, if it is one.
    pub fn as_include(&self) -> Option<&Include> {
        match self {
            Directive::Include(include) => Some(include),
            _ => None,
        }
    }
}

/// A directive together with where it is.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SpannedDirective {
    pub directive: Directive,
    pub range: SourceRange,
    /// The directive's **own logical line** — [`SpannedDirective::range`] without the trivia it rides.
    ///
    /// The difference matters to a consumer that draws **regions** rather than text: `range` runs through the
    /// newline after the directive, and through any comment that follows it (a comment is trivia, so "up to the
    /// next token that is not trivia" walks over whole comment blocks). A fold built from `range` would therefore
    /// end on the line below — or, after an `#include` run with a comment block beneath it, several lines below.
    ///
    /// Derived here rather than by each consumer, because "where does this directive end" is the same question as
    /// "where does it begin" and this function is the one that knows it. A `\`-continued directive keeps its line:
    /// the splice is not a newline, which is what `scan_directives` already measures by.
    pub line: SourceRange,
    /// Nesting depth of the conditional regions this directive sits inside.
    ///
    /// `0` at file scope. A consumer drawing a file's structure — folding, an outline, the "which
    /// `#if` am I in" question — needs this and cannot derive it from the directive alone.
    pub condition_depth: usize,
}

/// Read every directive in a file, in source order.
///
/// **Read from the token stream, not from the tree**, because the rule is a rule about lines: a `#` begins
/// a directive when it is the first token of its **logical** line. A splice is not a newline —
/// `LineContinuation` is a token kind of its own — which is what keeps `#define F(a) \` continued on the
/// next line from starting a second directive at the `#` of a stringized parameter.
///
/// A directive's **span** is the one the tree gives it: from the `#` through the trivia that follows its
/// last token, up to the next token that is not trivia. That trailing trivia (the newline, blank lines,
/// the comment before the following line) belongs to the span because the parser attaches it to the
/// directive it just finished, and a consumer asking for "the text of this directive" has always got it.
pub fn scan_directives(source: &str, tokens: &[CppTokenData]) -> Vec<SpannedDirective> {
    let mut out = Vec::new();
    let mut depth = 0usize;
    let mut index = 0usize;

    while index < tokens.len() {
        if tokens[index].kind != CppTokenKind::Hash || !starts_its_logical_line(tokens, index) {
            index += 1;
            continue;
        }

        // The logical line ends at the first real newline; everything up to it — the `\`-newline splices
        // included — is this directive. Then the trivia after that newline rides along, for the reason the
        // span is described above.
        let mut end = index;
        while end < tokens.len() && tokens[end].kind != CppTokenKind::Newline {
            end += 1;
        }
        let mut after = if end < tokens.len() { end + 1 } else { end };
        while after < tokens.len() && is_trivia(tokens[after].kind) {
            after += 1;
        }

        let start_offset = tokens[index].range.start_offset;
        let end_offset = tokens
            .get(after)
            .map_or(source.len(), |token| token.range.start_offset);
        let range = SourceRange::new(start_offset, end_offset.saturating_sub(start_offset));

        // Where the directive's **own** text ends: back over the trivia that sits between its last real token and
        // the newline. That trivia is whitespace or a comment — neither is the directive, and both ride in `range`.
        let mut own = end;
        while own > index && is_trivia(tokens[own - 1].kind) {
            own -= 1;
        }
        let own_end = tokens
            .get(own.saturating_sub(1))
            .map_or(start_offset, |token| token.range.end_offset());
        let line = SourceRange::new(start_offset, own_end.saturating_sub(start_offset));

        let directive = parse_directive_tokens(&to_tokens(source, &tokens[index..after]), range);
        let kind = directive.kind();

        // `#endif` is the only directive that *ends* a region. `#elif` and `#else` continue the one they are
        // in — they are written at the same depth as the `#if` they belong to, and everything after them is
        // still inside it. Treating `#else` as a closer is a one-line mistake that corrupts the depth of every
        // directive after it in the file: the `#define` in an `#else` branch is recorded as being at file
        // scope, so a consumer looking for "which `#if` is this in" finds none.
        if kind == DirectiveKind::Endif {
            out.push(SpannedDirective {
                directive,
                range,
                line,
                // The depth it was written at, which is the one it closes.
                condition_depth: depth.saturating_sub(1),
            });
            depth = depth.saturating_sub(1);
            index = after.max(index + 1);
            continue;
        }

        out.push(SpannedDirective {
            directive,
            range,
            line,
            condition_depth: depth,
        });

        if kind.opens_a_condition() {
            depth += 1;
        }
        index = after.max(index + 1);
    }

    out
}

/// What a file's directives say, as three numbers — the cheapest answer to "did this edit change what the file
/// tells the files that include it".
///
/// # Two numbers, because two things depend on a directive
///
/// * `environment` — the directives' **text**, in order and nothing else. It is what an *includer* sees: which
///   macros come into force, which files are pulled in, under which conditions. It does not move when an edit
///   above a directive shifts it down the file, because an includer never asked where a directive is.
/// * `layout` — the same, **and where each directive starts**. It is what a translation unit's timeline is made
///   of: its events are `(file, offset)` positions, so an edit that shifts a directive has changed the timeline
///   even though it has changed no macro.
///
/// So typing in a function body — after the file's last directive — changes neither, and typing between two
/// directives changes `layout` only when it changes the *length* of what is above the later one. The comparison is
/// conservative in the safe direction: two equal signatures mean the directives are the same lines at the same
/// places, and two different ones only mean "assume so".
///
/// # …and `preamble_end`, which is what a translation unit is read from
///
/// The offset **after the file's last directive**, or `0` for a file with none. Everything a walk reads out of a
/// file — its `#include`s, its `#define`s, its conditions — is at or before this offset, so a file whose bytes up
/// to here have not moved has not moved *as an input to a unit*, however much of its body was typed into.
///
/// That is the whole reason the field exists: it is what lets a translation unit be reused while the file it is
/// rooted at is being edited, which is the case that matters and the one the cache used to miss on every keystroke
/// (`crate::tu_cache`). It is taken at the **last** directive rather than at clangd's bound — the first
/// declaration, with the `#define`s before it — because the last one is a superset and needs no patch
/// (`PreamblePatch`-equivalent) to re-apply what it left out. The price is that an edit above the last directive
/// still invalidates, which is the conservative direction.
///
/// A directive's own text is its logical line without the trailing comment: a comment is trivia, and an edit to
/// one is not an edit to the directive.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct DirectiveSignature {
    pub environment: u64,
    pub layout: u64,
    pub preamble_end: usize,
}

/// Read the [`DirectiveSignature`] of `source`.
pub fn directive_signature(source: &str) -> DirectiveSignature {
    let (tokens, _) = cpp_parser::lex(source, &cpp_parser::LexerConfig::default());

    let mut environment = FNV_OFFSET;
    let mut layout = FNV_OFFSET;
    let mut preamble_end = 0;

    for directive in scan_directives(source, &tokens) {
        let line = directive.line;
        let text = source
            .get(line.start_offset..line.end_offset())
            .unwrap_or_default();

        preamble_end = preamble_end.max(line.end_offset());
        environment = fnv_mix(environment, text.as_bytes());
        // A separator, so that two lines are not the same as one line made of both.
        environment = fnv_mix(environment, &[0xff]);

        layout = fnv_mix(layout, &(line.start_offset as u64).to_le_bytes());
        layout = fnv_mix(layout, text.as_bytes());
        layout = fnv_mix(layout, &[0xff]);
    }

    DirectiveSignature {
        environment,
        layout,
        preamble_end,
    }
}

/// **Where a file's preamble ends** — after its last directive, or `0` for a file with none.
///
/// [`directive_signature`]'s third field, as a function, for the caller that wants only the bound
/// ([`crate::TranslationUnitCache`], which keys a unit's root on it). It scans, so a caller that wants the whole
/// signature should call `directive_signature` once rather than both.
pub fn preamble_end(source: &str) -> usize {
    directive_signature(source).preamble_end
}

const FNV_OFFSET: u64 = 0xcbf2_9ce4_8422_2325;

fn fnv_mix(mut hash: u64, bytes: &[u8]) -> u64 {
    for byte in bytes {
        hash ^= u64::from(*byte);
        hash = hash.wrapping_mul(0x0000_0100_0000_01b3);
    }
    hash
}

/// Is this token the first thing on its **logical** line?
///
/// Scanned backwards over trivia. A `Newline` makes the answer yes; any other token makes it no; reaching
/// the start of the file makes it yes. A `LineContinuation` is trivia and is deliberately *not* a newline:
/// a directive cannot begin on a line that the previous line was spliced onto.
fn starts_its_logical_line(tokens: &[CppTokenData], index: usize) -> bool {
    let mut i = index;
    while i > 0 {
        i -= 1;
        if tokens[i].kind == CppTokenKind::Newline {
            return true;
        }
        if !is_trivia(tokens[i].kind) {
            return false;
        }
    }
    true
}

/// The tokens of a span, with their text taken out of the source — what [`parse_directive_tokens`] reads.
pub(super) fn to_tokens(source: &str, tokens: &[CppTokenData]) -> Vec<Token> {
    tokens
        .iter()
        .map(|token| {
            let text = &source[token.range.start_offset..token.range.end_offset()];
            Token::new(token.kind, text, token.range)
        })
        .collect()
}

/// Read one directive from its tokens, including the leading `#`.
pub fn parse_directive_tokens(tokens: &[Token], range: SourceRange) -> Directive {
    let mut index = 0;

    // Leading whitespace and comments before the `#` are legal.
    skip_trivia(tokens, &mut index);

    match tokens.get(index).map(|token| token.kind) {
        Some(CppTokenKind::Hash) => index += 1,
        // The caller handed over tokens that are not a directive. `Null` is the honest answer: it is
        // what a bare `#` is, and it carries no arguments to misread.
        _ => return Directive::Null,
    }

    skip_trivia(tokens, &mut index);

    let Some(name) = tokens.get(index) else {
        // `#` alone on a line.
        return Directive::Null;
    };
    let kind = DirectiveKind::from_name(name.text());
    index += 1;

    // The rest of the directive, with the layout before it removed. Trivia *inside* the arguments is
    // kept: `#define A (1)` and `#define A(1)` differ only by whitespace, and so does a condition with
    // a line splice in it.
    let rest = strip_leading_trivia(&tokens[index.min(tokens.len())..]);

    match kind {
        DirectiveKind::If | DirectiveKind::Elif => Directive::Conditional {
            kind,
            condition: significant(rest),
        },
        DirectiveKind::Ifdef | DirectiveKind::Ifndef => match first_identifier(rest) {
            Some(name) => Directive::Ifdef { kind, name },
            None => Directive::Other {
                name: kind.name().into(),
            },
        },
        DirectiveKind::Else | DirectiveKind::Endif => Directive::Marker { kind },
        DirectiveKind::Define => match parse_define(rest, range) {
            Some(macro_def) => Directive::Define(Define {
                macro_def: Some(macro_def),
                // Redefinition is decided by the caller, which is the one holding the table.
                is_redefinition: false,
            }),
            // A `#define` whose arguments did not read is still a `#define`. Saying "unknown
            // directive" would lose the one fact that matters about the line.
            None => Directive::Define(Define {
                macro_def: None,
                is_redefinition: false,
            }),
        },
        DirectiveKind::Undef => Directive::Undef {
            name: first_identifier(rest),
        },
        DirectiveKind::Include | DirectiveKind::IncludeNext => parse_include(rest, kind),
        DirectiveKind::Pragma => Directive::Pragma {
            tokens: significant(rest),
        },
        DirectiveKind::Error | DirectiveKind::Warning => Directive::Diagnostic {
            kind,
            message: message_text(rest),
        },
        DirectiveKind::Line => Directive::Line {
            tokens: significant(rest),
        },
        DirectiveKind::Null | DirectiveKind::Unknown => Directive::Other {
            name: name.text.as_str().into(),
        },
    }
}

/// Read the target of an `#include`.
fn parse_include(tokens: &[Token], kind: DirectiveKind) -> Directive {
    let is_next = kind == DirectiveKind::IncludeNext;

    let Some(first) = tokens.iter().find(|token| !is_trivia(token.kind)) else {
        return Directive::Other {
            name: kind.name().into(),
        };
    };

    // The lexer folds `<vector>` into one token when the parser asks it to, so both forms arrive as a
    // `HeaderName`. Reading the delimiters back out is still necessary for the *form*: whether a
    // header is searched relative to the including file first is the whole difference between the two
    // spellings, and the tree does not record it separately.
    let (form, target) = match first.kind {
        CppTokenKind::HeaderName => {
            let text = first.text();
            if text.starts_with('<') {
                (
                    IncludeForm::Angle,
                    text.trim_start_matches('<').trim_end_matches('>'),
                )
            } else {
                (
                    IncludeForm::Quote,
                    text.trim_start_matches('"').trim_end_matches('"'),
                )
            }
        }
        // `#include <a/b.h>` that the lexer did not fold — the tokens are still there, and the text
        // between them is the name. Reconstructing it from tokens rather than from a range keeps this
        // working for a header name that was never folded.
        //
        // **After** the `<`, not from it: `tokens` begins with the delimiter, and appending from there
        // produced `<bits/c++config.h` as the target — which resolves to nothing, so every file that
        // included one was built, found an unresolved include, and was deliberately not cached (see
        // `index::store`). The trivia between the delimiters is kept, because the text between them *is*
        // the name: a space is a character a header name may contain, and dropping it would make this
        // disagree with the compiler about which file was meant.
        CppTokenKind::Less => {
            let mut text = String::new();
            for token in tokens
                .iter()
                .skip_while(|token| token.kind != CppTokenKind::Less)
                .skip(1)
            {
                if token.kind == CppTokenKind::Greater {
                    return Directive::Include(Include {
                        form: IncludeForm::Angle,
                        target: text.into(),
                        is_next,
                    });
                }
                text.push_str(token.text());
            }
            return Directive::Other {
                name: kind.name().into(),
            };
        }
        // `#include HEADER`, where `HEADER` is a macro. Well formed; the target is not a literal.
        CppTokenKind::Identifier => {
            return Directive::Include(Include {
                form: IncludeForm::Macro,
                target: first.text.as_str().into(),
                is_next,
            });
        }
        _ => {
            return Directive::Other {
                name: kind.name().into(),
            };
        }
    };

    Directive::Include(Include {
        form,
        target: target.into(),
        is_next,
    })
}

/// The message of an `#error` or `#warning`, joined by single spaces.
///
/// The text as written rather than a token list: an error message is for a person to read, and every
/// consumer of it wants the string. Whitespace is normalised because the message is a diagnostic, and
/// a diagnostic whose rendering depends on the author's indentation is a worse diagnostic.
fn message_text(tokens: &[Token]) -> String {
    let mut out = String::new();
    for token in tokens.iter().filter(|token| !is_trivia(token.kind)) {
        if !out.is_empty() {
            out.push(' ');
        }
        out.push_str(token.text());
    }
    out
}

fn first_identifier(tokens: &[Token]) -> Option<Box<str>> {
    tokens
        .iter()
        .find(|token| !is_trivia(token.kind))
        .filter(|token| token.is_identifier())
        .map(|token| token.text.as_str().into())
}

fn significant(tokens: &[Token]) -> Vec<Token> {
    tokens
        .iter()
        .filter(|token| !is_trivia(token.kind))
        .cloned()
        .collect()
}

fn strip_leading_trivia(tokens: &[Token]) -> &[Token] {
    let mut index = 0;
    skip_trivia(tokens, &mut index);
    &tokens[index..]
}

fn skip_trivia(tokens: &[Token], index: &mut usize) {
    while tokens
        .get(*index)
        .is_some_and(|token| is_trivia(token.kind))
    {
        *index += 1;
    }
}

/// Give `#include "local.h"` the token the parser gives it: a header name.
///
/// The parser folds the name after `#include` into one `HeaderName` token while it parses, because only the parser
/// knows a header name is expected there; the lexer alone leaves a quoted one as a string literal, and the directive
/// reader — which reads `HeaderName` — would call the line an unreadable directive. A scan that runs before the parse
/// has to do the same relabelling itself, by the same rule the parser uses (no escapes: a header name has none). The
/// angle form needs nothing: this reader already reassembles `<a/b.h>` from the tokens between the brackets.
///
/// **One implementation for the two readers.** It began in the indexer, for the directive summary; the include walk
/// needs it too, now that a header's directives are read from its tokens rather than from a parse. Two copies would
/// be two answers to "what does this `#include` line say", and a file one of them resolved and the other did not see
/// is a difference no test would attribute.
pub(crate) fn fold_quoted_header_names(source: &str, tokens: &mut [CppTokenData]) {
    let next_significant = |tokens: &[CppTokenData], from: usize| {
        (from..tokens.len()).find(|&at| !cpp_parser::is_trivia(tokens[at].kind))
    };

    for at in 0..tokens.len() {
        if tokens[at].kind != CppTokenKind::Hash {
            continue;
        }
        let Some(name) = next_significant(tokens, at + 1) else {
            continue;
        };
        let word = &source[tokens[name].range.start_offset..tokens[name].range.end_offset()];
        if word != "include" && word != "include_next" {
            continue;
        }
        let Some(target) = next_significant(tokens, name + 1) else {
            continue;
        };
        if tokens[target].kind != CppTokenKind::StringLiteral {
            continue;
        }
        let text = &source[tokens[target].range.start_offset..tokens[target].range.end_offset()];
        if !text.contains('\\') {
            tokens[target].kind = CppTokenKind::HeaderName;
        }
    }
}