formal-ai 0.302.0

Formal symbolic AI implementation with OpenAI-compatible APIs
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
//! Issue #673: the self-AST census widened from one pinned module to the whole
//! `src/` workspace.
//!
//! [`crate::agentic_coding::self_ast`] stores the CST/AST of *one* module
//! (`src/agentic_coding/planner.rs`, requirement R381). A planner that can only
//! introspect a single file cannot plan an edit anywhere else, so this module
//! generalizes that recipe to a **target set**: every owned Rust module reported
//! by the compile-time manifest ([`crate::self_source_links::owned_source_files`],
//! generated by `build.rs`) gets its own census document, plus one workspace index
//! that addresses them all.
//!
//! Two honest fidelity tiers keep the committed seed small without dropping a
//! module:
//!
//! * **`full_ast`** — modules under [`FULL_FIDELITY_PREFIX`] carry the complete
//!   abstract-syntax node-kind histogram produced by the sole CST/AST engine here
//!   (the meta-language links network), exactly like the pinned R381 artifact.
//! * **`signature`** — every other module carries its item/symbol/span table only.
//!
//! The tier is recorded in the document as a `fidelity` marker, so a reader never
//! mistakes a signature census for a full parse.
//!
//! Everything here is a pure function of the embedded sources: the same input
//! always renders the same bytes, which is what makes the committed census
//! *drift-checkable* ([`drift_report`]) the way the issue #559 method registry is
//! checked against the live dispatch table. Regeneration is per module, so a change
//! to one module re-censuses one document.

use std::collections::BTreeMap;
use std::fmt::Write as _;
use std::sync::OnceLock;

use crate::agentic_coding::self_ast::{ast_census, AstCensus};
use crate::engine::stable_id;
use crate::self_source_links::owned_source_files;

/// Directory holding the committed per-module census documents.
pub const CENSUS_DIR: &str = "data/meta/self-ast";

/// The committed workspace index that addresses every census document.
pub const INDEX_PATH: &str = "data/meta/self-ast/index.lino";

/// Modules under this prefix are censused at full AST fidelity.
///
/// The agentic-coding tree is the self-coding surface (E35/E36): it is the code
/// that plans edits to the rest of the workspace, so it is the part the planner
/// most needs a complete parse of.
pub const FULL_FIDELITY_PREFIX: &str = "src/agentic_coding/";

/// How much of a module the committed census records.
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub enum CensusFidelity {
    /// Item/symbol/span table plus the complete abstract-syntax node-kind census.
    FullAst,
    /// Item/symbol/span table only — no node-kind histogram.
    Signature,
}

impl CensusFidelity {
    /// The stable marker written into the census document.
    #[must_use]
    pub const fn slug(self) -> &'static str {
        match self {
            Self::FullAst => "full_ast",
            Self::Signature => "signature",
        }
    }

    /// Parse a marker back out of a committed document.
    #[must_use]
    pub fn from_slug(slug: &str) -> Option<Self> {
        match slug {
            "full_ast" => Some(Self::FullAst),
            "signature" => Some(Self::Signature),
            _ => None,
        }
    }
}

/// The fidelity tier a module path belongs to.
#[must_use]
pub fn fidelity_for(module_path: &str) -> CensusFidelity {
    if module_path.starts_with(FULL_FIDELITY_PREFIX) {
        CensusFidelity::FullAst
    } else {
        CensusFidelity::Signature
    }
}

/// The committed census document path for an owned module path.
///
/// `src/agentic_coding/planner.rs` → `data/meta/self-ast/src/agentic_coding/planner.lino`.
/// The mapping is injective and reversible ([`module_path_for_document`]), so the
/// index never needs to store both directions.
#[must_use]
pub fn document_path_for(module_path: &str) -> String {
    let stem = module_path.strip_suffix(".rs").unwrap_or(module_path);
    format!("{CENSUS_DIR}/{stem}.lino")
}

/// The owned module path a committed census document describes.
#[must_use]
pub fn module_path_for_document(document_path: &str) -> Option<String> {
    let inner = document_path.strip_prefix(CENSUS_DIR)?.strip_prefix('/')?;
    let stem = inner.strip_suffix(".lino")?;
    Some(format!("{stem}.rs"))
}

/// One named item recovered from a module, with the source lines it spans.
///
/// Spans are 1-based and inclusive, so `start_line == end_line` for a one-line
/// item and the pair addresses the exact text an edit would rewrite.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SymbolSpan {
    /// The item kind (`function`, `struct`, `enum`, `trait`, `type`, `module`,
    /// `const`, `static`, `union`, `macro`).
    pub kind: String,
    /// The declared identifier.
    pub name: String,
    /// First source line of the declaration (1-based, inclusive).
    pub start_line: usize,
    /// Last source line of the item (1-based, inclusive).
    pub end_line: usize,
}

/// The census of one owned module.
///
/// The signature tier — path, size, content id, and declared symbols — is
/// computed eagerly and is cheap (a lexical scan). The full abstract-syntax tier
/// is *deferred*: parsing a module through the meta-language links network costs
/// orders of magnitude more, and a planner resolving an edit target only needs
/// symbols. [`ModuleCensus::ast`] computes it on first use and memoizes it, so
/// rendering the committed documents pays for it while an interactive lookup
/// does not.
#[derive(Debug, Clone)]
pub struct ModuleCensus {
    /// Repository-relative path of the censused module.
    pub path: String,
    /// Which tier this census was taken at.
    pub fidelity: CensusFidelity,
    /// Byte length of the module source.
    pub byte_len: usize,
    /// Line count of the module source.
    pub line_count: usize,
    /// Content-addressed id of the module source — the drift signal.
    pub content_id: String,
    /// Declared items, in source order.
    pub symbols: Vec<SymbolSpan>,
    /// The module source, kept so the deferred AST tier can still be taken.
    source: String,
    /// Memoized abstract-syntax census; see [`ModuleCensus::ast`].
    ast: OnceLock<Option<AstCensus>>,
}

impl PartialEq for ModuleCensus {
    /// Two censuses are equal when they describe the same module at the same
    /// content id — the deferred AST is a pure function of that source, so it is
    /// not part of the identity.
    fn eq(&self, other: &Self) -> bool {
        self.path == other.path
            && self.fidelity == other.fidelity
            && self.content_id == other.content_id
            && self.symbols == other.symbols
    }
}

impl Eq for ModuleCensus {}

impl ModuleCensus {
    /// Census `source`, identified by its repository-relative `path`.
    ///
    /// The tier is chosen by [`fidelity_for`], so the caller cannot accidentally
    /// commit a full parse for a module the index advertises as signature-level.
    #[must_use]
    pub fn of(path: impl Into<String>, source: &str) -> Self {
        let path = path.into();
        let fidelity = fidelity_for(&path);
        Self {
            byte_len: source.len(),
            line_count: source.lines().count(),
            content_id: stable_id("source_module", source),
            symbols: scan_symbols(source),
            source: source.to_owned(),
            ast: OnceLock::new(),
            fidelity,
            path,
        }
    }

    /// The abstract-syntax node census, present only at
    /// [`CensusFidelity::FullAst`]. Computed on first use and memoized.
    #[must_use]
    pub fn ast(&self) -> Option<&AstCensus> {
        self.ast
            .get_or_init(|| match self.fidelity {
                CensusFidelity::FullAst => Some(ast_census(&self.source)),
                CensusFidelity::Signature => None,
            })
            .as_ref()
    }

    /// The committed document path for this census.
    #[must_use]
    pub fn document_path(&self) -> String {
        document_path_for(&self.path)
    }

    /// The first declared item named `name`, if this module declares one.
    #[must_use]
    pub fn symbol(&self, name: &str) -> Option<&SymbolSpan> {
        self.symbols.iter().find(|symbol| symbol.name == name)
    }

    /// Render the committed census document (Links Notation, one trailing newline).
    #[must_use]
    pub fn links_notation(&self) -> String {
        let mut out = String::new();
        let _ = writeln!(out, "self_ast_census");
        let _ = writeln!(out, "  target {}", self.path);
        let _ = writeln!(out, "  language rust");
        let _ = writeln!(out, "  fidelity {}", self.fidelity.slug());
        let _ = writeln!(out, "  byte_len {}", self.byte_len);
        let _ = writeln!(out, "  line_count {}", self.line_count);
        let _ = writeln!(out, "  content_id {}", self.content_id);
        let _ = writeln!(out, "  symbol_count {}", self.symbols.len());
        if !self.symbols.is_empty() {
            let _ = writeln!(out, "  symbols");
            for symbol in &self.symbols {
                let _ = writeln!(
                    out,
                    "    {} {} {} {}",
                    symbol.kind, symbol.name, symbol.start_line, symbol.end_line
                );
            }
        }
        if let Some(ast) = self.ast() {
            let _ = writeln!(out, "  ast");
            let _ = writeln!(out, "    engine meta_language");
            let _ = writeln!(out, "    projection abstract_syntax");
            let _ = writeln!(out, "    text_preserved {}", ast.text_preserved);
            let _ = writeln!(out, "    clean {}", ast.clean);
            let _ = writeln!(out, "    total_link_count {}", ast.total_link_count);
            let _ = writeln!(out, "    named_node_count {}", ast.named_node_count);
            let _ = writeln!(out, "    distinct_node_kinds {}", ast.node_kinds.len());
            if !ast.node_kinds.is_empty() {
                let _ = writeln!(out, "    node_kinds");
                for node in &ast.node_kinds {
                    let _ = writeln!(out, "      {} {}", node.kind, node.count);
                }
            }
        }
        out
    }
}

/// Where a `path:symbol` reference lands in the workspace census.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CensusResolution {
    /// The owned module the reference resolves to.
    pub module_path: String,
    /// The committed census document describing that module.
    pub document_path: String,
    /// The tier that document was censused at.
    pub fidelity: CensusFidelity,
    /// The resolved item, when the reference named one.
    pub symbol: Option<SymbolSpan>,
}

impl CensusResolution {
    /// The canonical `path:symbol` (or bare `path`) form of this resolution.
    #[must_use]
    pub fn reference(&self) -> String {
        self.symbol.as_ref().map_or_else(
            || self.module_path.clone(),
            |symbol| format!("{}:{}", self.module_path, symbol.name),
        )
    }
}

/// The census of every owned module, plus the index that addresses them.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct WorkspaceCensus {
    /// Per-module censuses, ordered by module path.
    pub modules: Vec<ModuleCensus>,
}

impl WorkspaceCensus {
    /// Census an explicit `(path, source)` target set.
    ///
    /// Taking the target set as an argument is what makes the recipe general: the
    /// committed census uses the whole owned manifest, while a test can hand it a
    /// three-file fixture and get the same documents and index shape back.
    #[must_use]
    pub fn compile(files: &[(&str, &str)]) -> Self {
        let mut modules: Vec<ModuleCensus> = files
            .iter()
            .map(|(path, source)| ModuleCensus::of(*path, source))
            .collect();
        modules.sort_by(|a, b| a.path.cmp(&b.path));
        Self { modules }
    }

    /// How many modules the census covers.
    #[must_use]
    pub const fn module_count(&self) -> usize {
        self.modules.len()
    }

    /// How many modules carry a full abstract-syntax census.
    #[must_use]
    pub fn full_fidelity_count(&self) -> usize {
        self.modules
            .iter()
            .filter(|module| module.fidelity == CensusFidelity::FullAst)
            .count()
    }

    /// How many declared items the census addresses across the workspace.
    #[must_use]
    pub fn symbol_count(&self) -> usize {
        self.modules.iter().map(|module| module.symbols.len()).sum()
    }

    /// The census of an exact module path.
    #[must_use]
    pub fn module(&self, module_path: &str) -> Option<&ModuleCensus> {
        self.modules
            .iter()
            .find(|module| module.path == module_path)
    }

    /// Resolve a module reference: an exact path, or an unambiguous path suffix
    /// such as `source_links.rs` or `agentic_coding/source_links.rs`.
    ///
    /// Ambiguous suffixes (`mod.rs`) resolve to nothing rather than to an
    /// arbitrary module — a planner must never guess which file it is editing.
    #[must_use]
    pub fn module_by_reference(&self, reference: &str) -> Option<&ModuleCensus> {
        if let Some(exact) = self.module(reference) {
            return Some(exact);
        }
        let suffix = format!("/{reference}");
        let mut matches = self
            .modules
            .iter()
            .filter(|module| module.path.ends_with(&suffix));
        let first = matches.next()?;
        matches.next().is_none().then_some(first)
    }

    /// The modules that declare an item named `name`.
    #[must_use]
    pub fn modules_declaring(&self, name: &str) -> Vec<&ModuleCensus> {
        self.modules
            .iter()
            .filter(|module| module.symbol(name).is_some())
            .collect()
    }

    /// Resolve a census reference to a module (and item, when named).
    ///
    /// Accepted forms, in order of precedence:
    ///
    /// * `src/agentic_coding/source_links.rs:render_document` — module and item;
    /// * `source_links.rs:render_document` — unambiguous module suffix and item;
    /// * `src/agentic_coding/source_links.rs` / `source_links.rs` — module only;
    /// * `render_document` — a bare item name, resolved only when exactly one
    ///   module in the workspace declares it.
    ///
    /// Every form fails closed: an unknown or ambiguous reference is `None`, never
    /// a guess.
    #[must_use]
    pub fn resolve(&self, reference: &str) -> Option<CensusResolution> {
        let reference = reference.trim();
        if reference.is_empty() {
            return None;
        }
        if let Some((module_reference, symbol_name)) = reference.rsplit_once(':') {
            let module = self.module_by_reference(module_reference)?;
            let symbol = module.symbol(symbol_name)?;
            return Some(resolution(module, Some(symbol.clone())));
        }
        if let Some(module) = self.module_by_reference(reference) {
            return Some(resolution(module, None));
        }
        let declaring = self.modules_declaring(reference);
        let [module] = declaring[..] else {
            return None;
        };
        let symbol = module.symbol(reference)?.clone();
        Some(resolution(module, Some(symbol)))
    }

    /// Every committed artifact this census owns: the per-module documents plus
    /// the workspace index, as `(path, contents)` pairs in committed order.
    #[must_use]
    pub fn documents(&self) -> Vec<(String, String)> {
        let mut documents: Vec<(String, String)> = self
            .modules
            .iter()
            .map(|module| (module.document_path(), module.links_notation()))
            .collect();
        documents.push((INDEX_PATH.to_owned(), self.index_notation()));
        documents.sort_by(|a, b| a.0.cmp(&b.0));
        documents
    }

    /// A content-addressed id over the whole census — one value that changes when
    /// any module's content id or fidelity changes.
    #[must_use]
    pub fn content_id(&self) -> String {
        let mut joined = String::new();
        for module in &self.modules {
            let _ = writeln!(
                joined,
                "{} {} {}",
                module.path,
                module.fidelity.slug(),
                module.content_id
            );
        }
        stable_id("self_ast_census_index", &joined)
    }

    /// Render the committed workspace index (Links Notation, one trailing newline).
    #[must_use]
    pub fn index_notation(&self) -> String {
        let mut out = String::new();
        let _ = writeln!(out, "self_ast_census_index");
        let _ = writeln!(out, "  language rust");
        let _ = writeln!(out, "  engine meta_language");
        let _ = writeln!(out, "  full_fidelity_prefix {FULL_FIDELITY_PREFIX}");
        let _ = writeln!(out, "  module_count {}", self.module_count());
        let _ = writeln!(
            out,
            "  full_ast_module_count {}",
            self.full_fidelity_count()
        );
        let _ = writeln!(
            out,
            "  signature_module_count {}",
            self.module_count() - self.full_fidelity_count()
        );
        let _ = writeln!(out, "  symbol_count {}", self.symbol_count());
        let _ = writeln!(out, "  content_id {}", self.content_id());
        let _ = writeln!(out, "  modules");
        for module in &self.modules {
            let _ = writeln!(
                out,
                "    {} {} {} {}",
                module.path,
                module.fidelity.slug(),
                module.symbols.len(),
                module.content_id
            );
        }
        out
    }
}

/// Build a resolution record for a module and an optionally named item.
fn resolution(module: &ModuleCensus, symbol: Option<SymbolSpan>) -> CensusResolution {
    CensusResolution {
        module_path: module.path.clone(),
        document_path: module.document_path(),
        fidelity: module.fidelity,
        symbol,
    }
}

/// The workspace census of the owned source manifest, computed once per process.
///
/// The full-fidelity tier parses real Rust through the meta-language engine, so
/// this is memoized exactly like [`crate::self_source_links::SourceLinks::owned`].
#[must_use]
pub fn workspace() -> &'static WorkspaceCensus {
    static WORKSPACE: OnceLock<WorkspaceCensus> = OnceLock::new();
    WORKSPACE.get_or_init(|| WorkspaceCensus::compile(owned_source_files()))
}

/// One way a committed census can diverge from the source it describes.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum CensusDrift {
    /// The source has a module with no committed census document.
    Missing {
        /// The census document that should exist.
        document: String,
    },
    /// The committed document no longer matches what the source renders.
    Stale {
        /// The census document whose bytes are out of date.
        document: String,
    },
    /// A committed document describes a module that no longer exists.
    Orphan {
        /// The census document that should be deleted.
        document: String,
    },
}

impl CensusDrift {
    /// The census document this finding is about.
    #[must_use]
    pub fn document(&self) -> &str {
        match self {
            Self::Missing { document } | Self::Stale { document } | Self::Orphan { document } => {
                document
            }
        }
    }

    /// The kind of divergence, as a stable slug.
    #[must_use]
    pub const fn kind(&self) -> &'static str {
        match self {
            Self::Missing { .. } => "missing",
            Self::Stale { .. } => "stale",
            Self::Orphan { .. } => "orphan",
        }
    }

    /// A one-line finding for a failing drift check, as `kind document`.
    ///
    /// Deliberately not prose: this is developer-facing diagnostic data in the
    /// same shape as the census documents themselves, so it carries no
    /// user-facing natural language (R379).
    #[must_use]
    pub fn describe(&self) -> String {
        format!("{} {}", self.kind(), self.document())
    }
}

/// Compare the census the sources render against the committed documents.
///
/// This is the drift guard the issue asks CI to run, in the shape the issue #559
/// method registry established: the authority is the live code, the committed
/// artifact is only a projection of it, and any divergence is a build failure.
/// Findings are ordered by document path so the failure message is stable.
#[must_use]
pub fn drift_report(
    expected: &[(String, String)],
    committed: &[(String, String)],
) -> Vec<CensusDrift> {
    let expected_by_path: BTreeMap<&str, &str> = expected
        .iter()
        .map(|(path, text)| (path.as_str(), text.as_str()))
        .collect();
    let committed_by_path: BTreeMap<&str, &str> = committed
        .iter()
        .map(|(path, text)| (path.as_str(), text.as_str()))
        .collect();

    let mut findings = Vec::new();
    for (path, text) in &expected_by_path {
        match committed_by_path.get(path) {
            None => findings.push(CensusDrift::Missing {
                document: (*path).to_owned(),
            }),
            Some(current) if current != text => findings.push(CensusDrift::Stale {
                document: (*path).to_owned(),
            }),
            Some(_) => {}
        }
    }
    for path in committed_by_path.keys() {
        if !expected_by_path.contains_key(path) {
            findings.push(CensusDrift::Orphan {
                document: (*path).to_owned(),
            });
        }
    }
    findings.sort_by(|a, b| a.document().cmp(b.document()));
    findings
}

/// Recover the declared items of a Rust module, with their inclusive line spans.
///
/// This is a deliberately *lexical* scan, not a parse: it is the signature tier's
/// engine, so it must work for every module even when the optional meta-language
/// feature is off, and it must never be more expensive than reading the file. It
/// tracks block comments, line comments, string and char literals so a brace
/// inside a literal cannot close an item early.
#[must_use]
pub fn scan_symbols(source: &str) -> Vec<SymbolSpan> {
    let lines: Vec<&str> = source.lines().collect();
    let mut code = Vec::with_capacity(lines.len());
    let mut in_block_comment = false;
    for line in &lines {
        code.push(strip_noncode(line, &mut in_block_comment));
    }

    let mut symbols = Vec::new();
    for (index, line) in code.iter().enumerate() {
        let Some((kind, name)) = declared_item(line) else {
            continue;
        };
        symbols.push(SymbolSpan {
            kind: kind.to_owned(),
            name,
            start_line: index + 1,
            end_line: item_end_line(&code, index),
        });
    }
    symbols
}

/// Blank out comments and literal contents in one source line, so brace counting
/// and keyword matching only ever see code.
fn strip_noncode(line: &str, in_block_comment: &mut bool) -> String {
    let bytes: Vec<char> = line.chars().collect();
    let mut out = String::with_capacity(line.len());
    let mut index = 0;
    while index < bytes.len() {
        let current = bytes[index];
        let next = bytes.get(index + 1).copied();
        if *in_block_comment {
            if current == '*' && next == Some('/') {
                *in_block_comment = false;
                index += 2;
            } else {
                index += 1;
            }
            continue;
        }
        match (current, next) {
            ('/', Some('/')) => break,
            ('/', Some('*')) => {
                *in_block_comment = true;
                index += 2;
            }
            ('"', _) => {
                out.push('"');
                index += 1;
                while index < bytes.len() {
                    if bytes[index] == '\\' {
                        index += 2;
                        continue;
                    }
                    if bytes[index] == '"' {
                        break;
                    }
                    index += 1;
                }
                index += 1;
                out.push('"');
            }
            ('\'', _) => {
                // A lifetime (`'a`) is code; a char literal (`'{'`) is not.
                let is_char_literal = matches!(bytes.get(index + 1), Some('\\'))
                    || matches!(bytes.get(index + 2), Some('\''));
                if is_char_literal {
                    let mut cursor = index + 1;
                    if bytes.get(cursor) == Some(&'\\') {
                        cursor += 1;
                    }
                    while cursor < bytes.len() && bytes[cursor] != '\'' {
                        cursor += 1;
                    }
                    index = cursor + 1;
                    out.push_str("''");
                } else {
                    out.push(current);
                    index += 1;
                }
            }
            _ => {
                out.push(current);
                index += 1;
            }
        }
    }
    out
}

/// The item kind and name declared on `line`, when it opens one.
fn declared_item(line: &str) -> Option<(&'static str, String)> {
    let mut tokens = line.split_whitespace().peekable();
    let mut first = *tokens.peek()?;
    if first.starts_with("pub") {
        // `pub`, `pub(crate)`, `pub(super)`, `pub(in path)` — the visibility may be
        // split across tokens (`pub(in crate::x)`), so consume until it closes.
        let mut visibility = tokens.next()?.to_owned();
        while visibility.contains('(') && !visibility.contains(')') {
            visibility.push_str(tokens.next()?);
        }
        first = *tokens.peek()?;
    }
    // Modifiers that may precede the item keyword.
    while matches!(first, "default" | "async" | "unsafe" | "extern") {
        let modifier = tokens.next()?;
        if modifier == "extern" {
            // `extern "C" fn` — the ABI string is not the item keyword.
            if tokens.peek().is_some_and(|token| token.starts_with('"')) {
                tokens.next();
            }
        }
        first = *tokens.peek()?;
    }
    if first == "const" {
        // `const fn` is a modifier; `const NAME:` is the item itself.
        let mut lookahead = tokens.clone();
        lookahead.next();
        if lookahead.peek() == Some(&"fn") {
            tokens.next();
            first = "fn";
        }
    }
    let kind = match first {
        "fn" => "function",
        "struct" => "struct",
        "enum" => "enum",
        "trait" => "trait",
        "type" => "type",
        "mod" => "module",
        "const" => "const",
        "static" => "static",
        "union" => "union",
        _ if first.starts_with("macro_rules!") => "macro",
        _ => return None,
    };
    tokens.next();
    if kind == "static" && tokens.peek() == Some(&"mut") {
        tokens.next();
    }
    let raw = if kind == "macro" {
        // `macro_rules! name {` and `macro_rules!name {` are both legal.
        let inline = first.trim_start_matches("macro_rules!");
        if inline.is_empty() {
            (*tokens.peek()?).to_owned()
        } else {
            inline.to_owned()
        }
    } else {
        (*tokens.peek()?).to_owned()
    };
    let name: String = raw
        .chars()
        .take_while(|character| character.is_alphanumeric() || *character == '_')
        .collect();
    let valid = !name.is_empty()
        && !name.starts_with(|character: char| character.is_ascii_digit())
        && name != "_";
    valid.then_some((kind, name))
}

/// The last line of the item that opens on `code[start]`.
///
/// A braced item ends where its brace depth returns to zero; a `;`-terminated one
/// (`mod x;`, `const N: usize = 1;`, `type A = B;`) ends on its own line.
fn item_end_line(code: &[String], start: usize) -> usize {
    let mut depth = 0_usize;
    let mut opened = false;
    for (offset, line) in code.iter().enumerate().skip(start) {
        for character in line.chars() {
            match character {
                '{' => {
                    depth += 1;
                    opened = true;
                }
                '}' => {
                    depth = depth.saturating_sub(1);
                    if opened && depth == 0 {
                        return offset + 1;
                    }
                }
                ';' if !opened && depth == 0 => return offset + 1,
                _ => {}
            }
        }
    }
    code.len().max(start + 1)
}