Skip to main content

g8_extractor/
lib.rs

1//! `g8-extractor` — annotation extraction for the G8.
2//!
3//! Detects `// @g8.<kind>(...)` magic-comment annotations (and `# @g8.<kind>(...)`
4//! for Python) across Rust, TypeScript, Python, and Go source files via the
5//! `ast-grep` CLI shell-out. Also parses `AGENTS.md`, `CLAUDE.md`, and `*.g8.md`
6//! sidecar files into `Intent` / `Plan` / `Decision` annotations via a native
7//! Markdown-section scanner.
8//!
9//! # Python comment syntax
10//!
11//! Python uses `#` as its comment prefix, not `//`. Both forms are accepted:
12//! - `# @g8.capability(name = "x")` — Python-native, recommended for Python files.
13//! - `// @g8.capability(name = "x")` — uniform cross-language form, also accepted.
14//!
15//! The ast-grep `comment` node kind rule for Python matches both because
16//! Python's tree-sitter grammar represents all comment text under `comment` nodes
17//! regardless of the leading character. The annotation parser strips either prefix.
18//!
19//! # Architecture
20//!
21//! ```text
22//! Extractor::scan_dir(root)
23//!   ├── walk via `ignore` crate (respects .gitignore)
24//!   ├── source files (.rs .ts .tsx .py .go)
25//!   │     └── ast-grep CLI shell-out (--json=stream, per R2 §6)
26//!   │           └── parse NDJSON → Vec<RawAstMatch>
27//!   │                 └── strip comment prefix → call g8_core::parse_annotation_grammar
28//!   └── sidecar files (AGENTS.md, CLAUDE.md, *.g8.md)
29//!         └── Markdown-section scanner (per ARCH §3.2 / Decision 4)
30//!               └── heading regex → IntentKind mapping
31//! ```
32//!
33//! Returns [`ScanResult`] with a deduplicated `Vec<Annotation>`, warnings, and stats.
34//!
35//! # Runtime dependency
36//!
37//! The `ast-grep` binary (`sg`) must be on `PATH` (or configured via
38//! [`Extractor::with_binary`]). Install with `cargo install ast-grep` or via
39//! Homebrew (`brew install ast-grep`). The extractor checks at construction time
40//! and returns [`ExtractorError::BinaryNotFound`] if absent.
41
42use std::collections::BTreeMap;
43use std::ffi::OsStr;
44use std::io::{BufRead, BufReader};
45use std::path::{Path, PathBuf};
46use std::process::{Command, Stdio};
47use std::time::Instant;
48
49use ignore::WalkBuilder;
50use regex::Regex;
51use serde::Deserialize;
52use sha2::{Digest, Sha256};
53use thiserror::Error;
54use tracing::{debug, instrument, warn};
55
56use g8_core::{
57    Annotation, AnnotationKind, AnnotationValue, IntentKind, IntentSourceKind, SourceLocation,
58};
59
60// ── Public error type ────────────────────────────────────────────────────────
61
62/// Errors produced by `g8-extractor`.
63#[derive(Debug, Error)]
64pub enum ExtractorError {
65    /// The `ast-grep` binary is not on `PATH` (or not at the configured path).
66    ///
67    /// Install with:
68    /// ```text
69    /// cargo install ast-grep
70    /// # or
71    /// brew install ast-grep
72    /// ```
73    #[error(
74        "ast-grep binary not found; install with `cargo install ast-grep` or `brew install ast-grep`"
75    )]
76    BinaryNotFound,
77
78    /// The `ast-grep` subprocess exited with a non-zero status.
79    #[error("ast-grep subprocess failed: {0}")]
80    SubprocessFailed(String),
81
82    /// A line of NDJSON from `ast-grep` could not be deserialized.
83    #[error("failed to parse ast-grep JSON output: {0}")]
84    JsonParse(String),
85
86    /// An annotation string was syntactically invalid.
87    #[error("invalid magic-comment annotation at {file}:{line}: {message}")]
88    InvalidAnnotation {
89        file: PathBuf,
90        line: u32,
91        message: String,
92    },
93
94    /// An I/O error occurred while spawning the subprocess or reading output.
95    #[error(transparent)]
96    Io(#[from] std::io::Error),
97}
98
99// ── Public output types ──────────────────────────────────────────────────────
100
101/// The result of a full directory-tree scan.
102#[derive(Debug, Default)]
103pub struct ScanResult {
104    /// All extracted annotations, deduplicated by (file, line, kind).
105    pub annotations: Vec<Annotation>,
106    /// Non-fatal warnings encountered during extraction.
107    pub warnings: Vec<ScanWarning>,
108    /// Scan statistics.
109    pub stats: ScanStats,
110}
111
112/// A non-fatal warning encountered during extraction.
113#[derive(Debug)]
114pub struct ScanWarning {
115    pub file: PathBuf,
116    pub line: Option<u32>,
117    pub message: String,
118}
119
120/// Aggregate statistics for a scan run.
121#[derive(Debug, Default)]
122pub struct ScanStats {
123    pub files_scanned: u32,
124    pub annotations_found: u32,
125    pub duration_ms: u64,
126}
127
128// ── Extractor ────────────────────────────────────────────────────────────────
129
130/// The main entry point for annotation extraction.
131///
132/// Construct with [`Extractor::new`] (auto-discovers `ast-grep` on `PATH`) or
133/// [`Extractor::with_binary`] (explicit binary path).
134pub struct Extractor {
135    rules_path: PathBuf,
136    ast_grep_bin: PathBuf,
137}
138
139/// The bundled ast-grep rules, embedded at compile time.
140///
141/// Embedded (rather than looked up via `env!("CARGO_MANIFEST_DIR")`) so the
142/// compiled binary is relocatable: a build-time path bakes the build machine's
143/// source checkout location into the binary, and every `scan` breaks the
144/// moment that checkout moves or the binary runs on another machine.
145const EMBEDDED_RULES: &str = include_str!("../rules/g8-all-languages.yaml");
146
147/// Write the embedded rules to a content-addressed file in the OS temp dir.
148///
149/// `ast-grep` runs as a subprocess, so it needs a real file on disk. The
150/// filename carries a digest of the content: concurrent g8 processes
151/// converge on the same file, a binary carrying newer rules never picks up a
152/// stale copy, and the write is skipped once the file exists. Write-then-rename
153/// keeps a concurrent reader from ever seeing a half-written file.
154fn materialize_embedded_rules() -> Result<PathBuf, ExtractorError> {
155    let digest = {
156        let mut hasher = Sha256::new();
157        hasher.update(EMBEDDED_RULES.as_bytes());
158        let full = hasher.finalize();
159        // 64 bits of the digest is plenty for a cache key.
160        full[..8]
161            .iter()
162            .map(|b| format!("{b:02x}"))
163            .collect::<String>()
164    };
165    let path = std::env::temp_dir().join(format!("g8-rules-{digest}.yaml"));
166    if !path.exists() {
167        let staging = path.with_extension(format!("yaml.{}.tmp", std::process::id()));
168        std::fs::write(&staging, EMBEDDED_RULES)?;
169        std::fs::rename(&staging, &path)?;
170    }
171    Ok(path)
172}
173
174impl Extractor {
175    /// Construct an `Extractor` using the bundled (compile-time embedded) rules
176    /// and `ast-grep` from `PATH`.
177    ///
178    /// Returns [`ExtractorError::BinaryNotFound`] if `ast-grep` cannot be located.
179    pub fn new() -> Result<Self, ExtractorError> {
180        Self::with_binary(materialize_embedded_rules()?, find_ast_grep()?)
181    }
182
183    /// Construct using a custom rules YAML path and `ast-grep` from `PATH`.
184    pub fn with_rules(rules_path: PathBuf) -> Result<Self, ExtractorError> {
185        Self::with_binary(rules_path, find_ast_grep()?)
186    }
187
188    /// Construct using explicit paths for both the rules file and the binary.
189    pub fn with_binary(rules_path: PathBuf, ast_grep_bin: PathBuf) -> Result<Self, ExtractorError> {
190        // Verify binary is executable.
191        Command::new(&ast_grep_bin)
192            .arg("--version")
193            .stdout(Stdio::null())
194            .stderr(Stdio::null())
195            .status()
196            .map_err(|_| ExtractorError::BinaryNotFound)?;
197        Ok(Self {
198            rules_path,
199            ast_grep_bin,
200        })
201    }
202
203    /// Scan an entire directory tree for G8 annotations.
204    ///
205    /// Uses the `ignore` crate to walk the tree (respects `.gitignore`).
206    /// Source files are processed via ast-grep; Markdown sidecars are processed
207    /// by the native section scanner.
208    ///
209    /// The file `.g8/INTENT_SUMMARY.md` is excluded from extraction to avoid
210    /// a feedback loop (per ARCHITECTURE.md §3.2).
211    #[instrument(skip(self), fields(root = %root.display()))]
212    pub fn scan_dir(&self, root: &Path) -> Result<ScanResult, ExtractorError> {
213        let start = Instant::now();
214        let mut result = ScanResult::default();
215
216        // Collect source files and sidecar files by walking the tree.
217        let mut source_files: Vec<PathBuf> = Vec::new();
218        let mut sidecar_files: Vec<PathBuf> = Vec::new();
219
220        let walker = WalkBuilder::new(root)
221            .standard_filters(true) // respects .gitignore, .ignore, etc.
222            .build();
223
224        for entry in walker.flatten() {
225            let path = entry.into_path();
226            if !path.is_file() {
227                continue;
228            }
229
230            // Exclude INTENT_SUMMARY.md (auto-generated; would create a loop).
231            if path
232                .file_name()
233                .and_then(OsStr::to_str)
234                .map(|n| n == "INTENT_SUMMARY.md")
235                .unwrap_or(false)
236            {
237                continue;
238            }
239
240            match classify_file(&path) {
241                FileKind::Source => source_files.push(path),
242                FileKind::Sidecar(_sk) => sidecar_files.push(path.clone()),
243                FileKind::Other => {}
244            }
245        }
246
247        result.stats.files_scanned = (source_files.len() + sidecar_files.len()) as u32;
248
249        // ── ast-grep pass for source files ──
250        if !source_files.is_empty() {
251            let ast_annotations = self.run_ast_grep(root, &mut result.warnings)?;
252            result.annotations.extend(ast_annotations);
253        }
254
255        // ── Markdown sidecar pass ──
256        for sidecar in &sidecar_files {
257            let sk = sidecar_source_kind(sidecar);
258            match parse_sidecar(sidecar, sk) {
259                Ok(mut anns) => result.annotations.append(&mut anns),
260                Err(e) => result.warnings.push(ScanWarning {
261                    file: sidecar.clone(),
262                    line: None,
263                    message: e.to_string(),
264                }),
265            }
266        }
267
268        // Deduplicate: same (file, line, kind) triple.
269        dedup_annotations(&mut result.annotations);
270
271        result.stats.annotations_found = result.annotations.len() as u32;
272        result.stats.duration_ms = start.elapsed().as_millis() as u64;
273        Ok(result)
274    }
275
276    /// Scan a single source file for magic-comment annotations.
277    ///
278    /// For Markdown sidecars, delegates to the native section parser.
279    /// For other file types, invokes `ast-grep` restricted to that one file.
280    #[instrument(skip(self), fields(file = %file.display()))]
281    pub fn scan_file(&self, file: &Path) -> Result<Vec<Annotation>, ExtractorError> {
282        match classify_file(file) {
283            FileKind::Source => {
284                let mut warnings = Vec::new();
285                self.run_ast_grep(file, &mut warnings)
286            }
287            FileKind::Sidecar(sk) => parse_sidecar(file, sk).map_err(ExtractorError::Io),
288            FileKind::Other => Ok(Vec::new()),
289        }
290    }
291
292    /// Invoke `ast-grep scan --rule <rules> --json=stream <target>` and parse output.
293    #[instrument(skip(self, warnings), fields(target = %target.display()))]
294    fn run_ast_grep(
295        &self,
296        target: &Path,
297        warnings: &mut Vec<ScanWarning>,
298    ) -> Result<Vec<Annotation>, ExtractorError> {
299        let mut child = Command::new(&self.ast_grep_bin)
300            .args(["scan", "--rule"])
301            .arg(&self.rules_path)
302            .arg("--json=stream")
303            .arg(target)
304            .stdout(Stdio::piped())
305            .stderr(Stdio::null()) // suppress progress/colour noise
306            .spawn()
307            .map_err(|_| ExtractorError::BinaryNotFound)?;
308
309        let stdout = child.stdout.take().expect("piped stdout");
310        let reader = BufReader::new(stdout);
311
312        let name_re = Regex::new(r#"name\s*=\s*"([^"]+)""#).expect("static regex");
313        let mut annotations = Vec::new();
314
315        for line in reader.lines() {
316            let line = line?;
317            let line = line.trim();
318            if line.is_empty() {
319                continue;
320            }
321            let raw: RawAstMatch = match serde_json::from_str(line) {
322                Ok(m) => m,
323                Err(e) => {
324                    warn!("failed to parse ast-grep NDJSON line: {e}");
325                    continue;
326                }
327            };
328            debug!(rule = ?raw.rule_id, file = %raw.file, "ast-grep match");
329
330            match convert_ast_match(&raw, &name_re) {
331                Ok(Some(ann)) => annotations.push(ann),
332                Ok(None) => {} // match did not produce a usable annotation
333                Err(e) => warnings.push(ScanWarning {
334                    file: PathBuf::from(&raw.file),
335                    line: Some(raw.range.start.line as u32 + 1),
336                    message: e,
337                }),
338            }
339        }
340
341        let status = child.wait()?;
342        // ast-grep exits 0 whether or not matches were found; non-zero means a real error.
343        if !status.success() {
344            return Err(ExtractorError::SubprocessFailed(format!(
345                "exit code: {}",
346                status.code().unwrap_or(-1)
347            )));
348        }
349
350        Ok(annotations)
351    }
352}
353
354// ── Also expose a top-level free function matching ARCH §3.2 `scan_project` ─
355
356/// Scan a directory tree for all G8 annotations.
357///
358/// This is a convenience wrapper around [`Extractor::new`] + [`Extractor::scan_dir`].
359/// Returns `Err` if the `ast-grep` binary is not found.
360pub fn scan_project(root: &Path) -> Result<Vec<Annotation>, ExtractorError> {
361    let extractor = Extractor::new()?;
362    extractor.scan_dir(root).map(|r| r.annotations)
363}
364
365// ── Internal: ast-grep JSON types ────────────────────────────────────────────
366
367/// Raw JSON object emitted by `ast-grep scan --json=stream`.
368#[derive(Debug, Deserialize)]
369#[serde(rename_all = "camelCase")]
370struct RawAstMatch {
371    text: String,
372    file: String,
373    range: RawRange,
374    #[serde(default)]
375    rule_id: Option<String>,
376    #[serde(default)]
377    #[allow(dead_code)]
378    language: Option<String>,
379}
380
381#[derive(Debug, Deserialize)]
382struct RawRange {
383    start: RawPos,
384    #[allow(dead_code)]
385    end: RawPos,
386}
387
388#[derive(Debug, Deserialize)]
389struct RawPos {
390    line: usize,
391    column: usize,
392}
393
394// ── Internal: file classification ────────────────────────────────────────────
395
396#[derive(Debug)]
397enum FileKind {
398    Source,
399    Sidecar(IntentSourceKind),
400    Other,
401}
402
403fn classify_file(path: &Path) -> FileKind {
404    let file_name = path.file_name().and_then(OsStr::to_str).unwrap_or_default();
405
406    // Exact filename matches first.
407    if file_name == "AGENTS.md" {
408        return FileKind::Sidecar(IntentSourceKind::AgentsMd);
409    }
410    if file_name == "CLAUDE.md" {
411        return FileKind::Sidecar(IntentSourceKind::ClaudeMd);
412    }
413
414    let ext = path.extension().and_then(OsStr::to_str).unwrap_or_default();
415
416    match ext {
417        "rs" | "ts" | "tsx" | "py" | "go" => FileKind::Source,
418        "md" | "markdown" => {
419            // *.g8.md sidecar: any .md file that isn't the auto-generated summary.
420            // We treat all other .md files (other than AGENTS.md/CLAUDE.md already
421            // handled above) as G8 sidecars if they match *.g8.md, otherwise Other.
422            let stem = path.file_stem().and_then(OsStr::to_str).unwrap_or_default();
423            if stem.ends_with(".g8")
424                || file_name.ends_with(".g8.md")
425                || file_name.ends_with(".govern.md")
426            {
427                FileKind::Sidecar(IntentSourceKind::G8Sidecar)
428            } else {
429                FileKind::Other
430            }
431        }
432        _ => FileKind::Other,
433    }
434}
435
436fn sidecar_source_kind(path: &Path) -> IntentSourceKind {
437    let file_name = path.file_name().and_then(OsStr::to_str).unwrap_or_default();
438    if file_name == "AGENTS.md" {
439        IntentSourceKind::AgentsMd
440    } else if file_name == "CLAUDE.md" {
441        IntentSourceKind::ClaudeMd
442    } else {
443        IntentSourceKind::G8Sidecar
444    }
445}
446
447// ── Internal: convert ast-grep match → Annotation ────────────────────────────
448
449/// Convert one raw ast-grep match into an `Annotation`, or `None` if the match
450/// does not contain a parseable G8 annotation.
451fn convert_ast_match(
452    raw: &RawAstMatch,
453    _name_re: &Regex, // kept for potential future use
454) -> Result<Option<Annotation>, String> {
455    // Strip comment prefixes to isolate the `@g8.<kind>(...)` text.
456    let text = strip_comment_prefixes(&raw.text);
457
458    // Find the @g8. annotation within the (potentially multiline) text.
459    let g8_start = match text.find("@g8.").or_else(|| text.find("@govern.")) {
460        Some(pos) => pos,
461        None => return Ok(None),
462    };
463    let annotation_text = &text[g8_start..];
464
465    // Parse the annotation grammar.
466    let raw_ann = match g8_core::parse_annotation_grammar(annotation_text) {
467        Ok(r) => r,
468        Err(e) => return Err(e.to_string()),
469    };
470
471    let kind = annotation_kind_from_str(&raw_ann.kind_str)?;
472
473    // Extract name (for Capability / Plan) and heading (for Intent).
474    let name = match kind {
475        AnnotationKind::Capability | AnnotationKind::Plan => raw_ann
476            .fields
477            .get("name")
478            .or_else(|| raw_ann.fields.get("title"))
479            .and_then(|v| {
480                if let AnnotationValue::String(s) = v {
481                    Some(s.clone())
482                } else {
483                    None
484                }
485            }),
486        _ => None,
487    };
488
489    let heading = if kind == AnnotationKind::Intent {
490        raw_ann
491            .fields
492            .get("description")
493            .or_else(|| raw_ann.fields.get("title"))
494            .and_then(|v| {
495                if let AnnotationValue::String(s) = v {
496                    Some(s.clone())
497                } else {
498                    None
499                }
500            })
501    } else {
502        None
503    };
504
505    // Detect conditional flag.
506    let conditional = is_conditional(&raw.text, &raw_ann.fields);
507
508    // Detect tests-dir flag.
509    let path = PathBuf::from(&raw.file);
510    let in_tests_dir = is_tests_dir(&path);
511    let conditional = conditional || in_tests_dir;
512
513    // Determine source_kind: inline comment.
514    let source_kind = IntentSourceKind::InlineComment;
515
516    // ast-grep reports 0-based lines; SourceLocation.line is 1-based.
517    let source = SourceLocation {
518        file: path,
519        line: raw.range.start.line as u32 + 1,
520        column: raw.range.start.column as u32,
521    };
522
523    Ok(Some(Annotation {
524        kind,
525        name,
526        heading,
527        fields: raw_ann.fields,
528        source,
529        source_kind,
530        conditional,
531        raw_text: annotation_text.to_string(),
532    }))
533}
534
535/// Strip `//` and `#` comment prefixes from each line of a multiline match,
536/// then join all lines with a single space to produce a flat annotation string
537/// ready for [`g8_core::parse_annotation_grammar`].
538///
539/// This handles the uniform `// @g8.` prefix (all languages) and the Python-native
540/// `# @g8.` prefix. Lines that do not start with a recognised comment prefix are
541/// preserved as-is (trimmed).
542///
543/// ## v0.1 behaviour (space-join)
544///
545/// Consecutive `//`-prefixed continuation lines are joined with a single space.
546/// This means the multi-line form:
547///
548/// ```text
549/// // @g8.capability(name = "http-fetch",
550/// //                 status = "in_flight")
551/// ```
552///
553/// becomes the flat string:
554///
555/// ```text
556/// @g8.capability(name = "http-fetch", status = "in_flight")
557/// ```
558///
559/// which the grammar parser handles correctly because whitespace between tokens is
560/// ignored (ARCHITECTURE.md §6.1).
561///
562/// ## v0.2 roadmap note
563///
564/// The space-join approach works for all well-formed multi-line annotations but
565/// may produce confusing error messages when a continuation line is malformed
566/// (the error position refers to the flat string, not the original line number).
567/// A token-aware multi-line splitter that preserves per-line source positions is
568/// tracked for v0.2 (SPEC §10 roadmap, "native language idioms").
569fn strip_comment_prefixes(text: &str) -> String {
570    text.lines()
571        .map(|line| {
572            let trimmed = line.trim_start();
573            if let Some(rest) = trimmed.strip_prefix("// ") {
574                rest
575            } else if let Some(rest) = trimmed.strip_prefix("//") {
576                rest
577            } else if let Some(rest) = trimmed.strip_prefix("# ") {
578                rest
579            } else if let Some(rest) = trimmed.strip_prefix('#') {
580                rest
581            } else {
582                trimmed
583            }
584        })
585        .collect::<Vec<_>>()
586        .join(" ")
587}
588
589/// Convert a kind string from `RawAnnotation` to `AnnotationKind`.
590fn annotation_kind_from_str(s: &str) -> Result<AnnotationKind, String> {
591    match s {
592        "capability" => Ok(AnnotationKind::Capability),
593        "intent" => Ok(AnnotationKind::Intent),
594        "convergence_test" => Ok(AnnotationKind::ConvergenceTest),
595        "decision" => Ok(AnnotationKind::Decision),
596        "plan" => Ok(AnnotationKind::Plan),
597        other => Err(format!("unknown annotation kind: {other}")),
598    }
599}
600
601/// Returns `true` if the annotation should be flagged as conditional.
602///
603/// Three triggers (ARCHITECTURE.md §6.3):
604/// 1. `dev_only = true` in the annotation fields.
605/// 2. (Rust only) the raw text contains `#[cfg(` or `#[cfg_attr(` on the preceding line.
606/// 3. File path contains a `tests`, `tests-*`, or `*-tests` directory component
607///    (handled by `is_tests_dir` separately).
608fn is_conditional(raw_text: &str, fields: &BTreeMap<String, AnnotationValue>) -> bool {
609    // Check dev_only field.
610    if let Some(AnnotationValue::Bool(true)) = fields.get("dev_only") {
611        return true;
612    }
613    // Check raw text for cfg_attr / cfg preceding the annotation.
614    if raw_text.contains("#[cfg(") || raw_text.contains("#[cfg_attr(") {
615        return true;
616    }
617    false
618}
619
620/// Returns `true` if the file path lives under a `tests`, `tests-*`, or `*-tests` directory.
621fn is_tests_dir(path: &Path) -> bool {
622    path.components().any(|c| {
623        if let std::path::Component::Normal(s) = c {
624            let s = s.to_string_lossy();
625            s == "tests" || s.starts_with("tests-") || s.ends_with("-tests")
626        } else {
627            false
628        }
629    })
630}
631
632// ── Internal: Markdown sidecar parser ────────────────────────────────────────
633
634/// Parse an `AGENTS.md`, `CLAUDE.md`, or `*.g8.md` file into `Annotation` records.
635///
636/// Each heading (`#` or `##`) starts a new section. The heading text is mapped
637/// to an `IntentKind` (or `Plan` for parked-ideas sections) via the locked table
638/// in ARCHITECTURE.md §3.2 / SPEC Decision 4.
639///
640/// # Precedence
641/// The `scope_depth` (directory depth from the file's location) is encoded in
642/// `Annotation::source.line = 0` + caller uses file path depth for ordering.
643/// Deeper files win when the planner queries intents (ORDER BY scope_depth DESC).
644///
645/// # Excluded file
646/// `.g8/INTENT_SUMMARY.md` is excluded by the caller (`scan_dir`) before this
647/// function is called.
648fn parse_sidecar(
649    path: &Path,
650    source_kind: IntentSourceKind,
651) -> Result<Vec<Annotation>, std::io::Error> {
652    let content = std::fs::read_to_string(path)?;
653    let mut annotations = Vec::new();
654
655    let mut current_heading: Option<String> = None;
656    let mut current_lines: Vec<String> = Vec::new();
657    let mut current_line_number: u32 = 0;
658    let mut section_start_line: u32 = 1;
659
660    let flush =
661        |heading: &str, lines: &[String], line_num: u32, annotations: &mut Vec<Annotation>| {
662            let description = lines.join("\n").trim().to_string();
663            let (kind, is_owner, is_parked) = classify_heading(heading);
664
665            if is_owner {
666                // Owner sections are parsed and merged into the previous intent.
667                // For simplicity in v0.1, emit as Unclassified Intent; the store
668                // layer picks up owner_agent / owner_team / owner_contact columns.
669                let mut fields = BTreeMap::new();
670                parse_owner_block(&description, &mut fields);
671                fields.insert(
672                    "heading".to_string(),
673                    AnnotationValue::String(heading.to_string()),
674                );
675                annotations.push(Annotation {
676                    kind: AnnotationKind::Intent,
677                    name: None,
678                    heading: Some(heading.to_string()),
679                    fields,
680                    source: SourceLocation {
681                        file: path.to_path_buf(),
682                        line: line_num,
683                        column: 0,
684                    },
685                    source_kind,
686                    conditional: false,
687                    raw_text: description,
688                });
689                return;
690            }
691
692            if is_parked {
693                // Parked Ideas: parse line-by-line `- <name>: <reason>` entries.
694                for (i, line) in lines.iter().enumerate() {
695                    let line = line.trim();
696                    if let Some(rest) = line.strip_prefix("- ") {
697                        let (title, reason) = parse_parked_line(rest);
698                        let mut fields = BTreeMap::new();
699                        fields.insert("title".to_string(), AnnotationValue::String(title.clone()));
700                        fields.insert(
701                            "status".to_string(),
702                            AnnotationValue::String("Parked".to_string()),
703                        );
704                        if let Some(r) = reason {
705                            fields.insert("parked_reason".to_string(), AnnotationValue::String(r));
706                        }
707                        annotations.push(Annotation {
708                            kind: AnnotationKind::Plan,
709                            name: Some(title),
710                            heading: None,
711                            fields,
712                            source: SourceLocation {
713                                file: path.to_path_buf(),
714                                line: line_num + i as u32,
715                                column: 0,
716                            },
717                            source_kind,
718                            conditional: false,
719                            raw_text: line.to_string(),
720                        });
721                    } else if !line.is_empty() {
722                        // Non-matching line — emit a trace-level warning via tracing.
723                        // We don't fail; warn!() per ARCH spec.
724                        tracing::warn!(
725                            file = %path.display(),
726                            "parked-ideas line does not match `- <name>: <reason>` form: {line}"
727                        );
728                    }
729                }
730                return;
731            }
732
733            // Normal intent section.
734            let mut fields = BTreeMap::new();
735            fields.insert(
736                "description".to_string(),
737                AnnotationValue::String(description.clone()),
738            );
739            fields.insert(
740                "heading".to_string(),
741                AnnotationValue::String(heading.to_string()),
742            );
743            if let AnnotationKind::Intent = AnnotationKind::Intent {
744                let kind_str = intent_kind_str(kind);
745                fields.insert(
746                    "intent_kind".to_string(),
747                    AnnotationValue::String(kind_str.to_string()),
748                );
749            }
750
751            annotations.push(Annotation {
752                kind: AnnotationKind::Intent,
753                name: None,
754                heading: Some(heading.to_string()),
755                fields,
756                source: SourceLocation {
757                    file: path.to_path_buf(),
758                    line: line_num,
759                    column: 0,
760                },
761                source_kind,
762                conditional: false,
763                raw_text: description,
764            });
765        };
766
767    for line in content.lines() {
768        current_line_number += 1;
769
770        // Detect headings: `## ` or `# `.
771        let heading_text = if let Some(h) = line.strip_prefix("## ") {
772            Some(h.trim())
773        } else {
774            line.strip_prefix("# ").map(|h| h.trim())
775        };
776
777        if let Some(h) = heading_text {
778            // Flush the previous section.
779            if let Some(prev_heading) = current_heading.take() {
780                flush(
781                    &prev_heading,
782                    &current_lines,
783                    section_start_line,
784                    &mut annotations,
785                );
786                current_lines.clear();
787            }
788            current_heading = Some(h.to_string());
789            section_start_line = current_line_number;
790        } else if current_heading.is_some() {
791            current_lines.push(line.to_string());
792        }
793    }
794
795    // Flush the last section.
796    if let Some(prev_heading) = current_heading.take() {
797        flush(
798            &prev_heading,
799            &current_lines,
800            section_start_line,
801            &mut annotations,
802        );
803    }
804
805    Ok(annotations)
806}
807
808/// Map a heading string to (IntentKind, is_owner_section, is_parked_section).
809///
810/// Case-insensitive. Per SPEC Decision 4 / ARCHITECTURE.md §3.2 locked table.
811fn classify_heading(heading: &str) -> (IntentKind, bool, bool) {
812    let h = heading.to_lowercase();
813    let h = h.trim();
814
815    // Owner section — parsed separately.
816    if matches!(h, "owner" | "ownership") {
817        return (IntentKind::Unclassified, true, false);
818    }
819
820    // Parked ideas section.
821    if matches!(h, "parked ideas" | "deferred") {
822        return (IntentKind::Unclassified, false, true);
823    }
824
825    // Active plans (ignored by extractor — those rows live in `plan` table).
826    if matches!(h, "active plans" | "in flight" | "in-flight") {
827        return (IntentKind::Unclassified, false, false);
828    }
829
830    let kind = if matches!(h, "architecture" | "design" | "overview") {
831        IntentKind::ArchitecturalScope
832    } else if matches!(h, "boundaries" | "never do" | "constraints") {
833        IntentKind::Boundary
834    } else if matches!(h, "commands" | "build" | "test" | "testing") {
835        IntentKind::Operational
836    } else if matches!(h, "stack" | "dependencies") {
837        IntentKind::TechStack
838    } else if h == "intent" {
839        IntentKind::ExplicitIntent
840    } else if matches!(h, "decisions" | "architectural decisions") {
841        // Decisions heading → emit as Intent::Unclassified for now;
842        // the store layer can promote to Decision rows.
843        IntentKind::Unclassified
844    } else {
845        IntentKind::Unclassified
846    };
847
848    (kind, false, false)
849}
850
851/// Convert `IntentKind` to its snake_case string representation.
852fn intent_kind_str(kind: IntentKind) -> &'static str {
853    match kind {
854        IntentKind::ArchitecturalScope => "architectural_scope",
855        IntentKind::Boundary => "boundary",
856        IntentKind::Operational => "operational",
857        IntentKind::TechStack => "tech_stack",
858        IntentKind::ExplicitIntent => "explicit_intent",
859        IntentKind::Unclassified => "unclassified",
860    }
861}
862
863/// Parse `# Owner` block content into a fields map.
864///
865/// Accepted lines:
866/// ```text
867/// agent: scorer-specialist
868/// team: core-ranker
869/// contact: #checkout-api-core
870/// ```
871fn parse_owner_block(content: &str, fields: &mut BTreeMap<String, AnnotationValue>) {
872    for line in content.lines() {
873        let line = line.trim();
874        if let Some(rest) = line.strip_prefix("agent:") {
875            fields.insert(
876                "owner_agent".to_string(),
877                AnnotationValue::String(rest.trim().to_string()),
878            );
879        } else if let Some(rest) = line.strip_prefix("team:") {
880            fields.insert(
881                "owner_team".to_string(),
882                AnnotationValue::String(rest.trim().to_string()),
883            );
884        } else if let Some(rest) = line.strip_prefix("contact:") {
885            fields.insert(
886                "owner_contact".to_string(),
887                AnnotationValue::String(rest.trim().to_string()),
888            );
889        }
890    }
891}
892
893/// Parse a `- <name>: <reason>` parked-ideas line.
894///
895/// Returns `(name, Some(reason))` or `(name, None)` if no colon separator.
896fn parse_parked_line(rest: &str) -> (String, Option<String>) {
897    if let Some(colon_pos) = rest.find(": ") {
898        let name = rest[..colon_pos].trim().to_string();
899        let reason = rest[colon_pos + 2..].trim().to_string();
900        (name, Some(reason))
901    } else {
902        (rest.trim().to_string(), None)
903    }
904}
905
906// ── Internal: deduplication ───────────────────────────────────────────────────
907
908fn dedup_annotations(annotations: &mut Vec<Annotation>) {
909    use std::collections::HashSet;
910    let mut seen: HashSet<(PathBuf, u32, AnnotationKind)> = HashSet::new();
911    annotations.retain(|a| {
912        let key = (a.source.file.clone(), a.source.line, a.kind);
913        seen.insert(key)
914    });
915}
916
917// ── Internal: find ast-grep binary ───────────────────────────────────────────
918
919/// Locate `ast-grep` on `PATH`, trying common install locations.
920fn find_ast_grep() -> Result<PathBuf, ExtractorError> {
921    // Try common known paths first, then fall back to PATH search.
922    let candidates = [
923        "/opt/homebrew/bin/ast-grep",
924        "/usr/local/bin/ast-grep",
925        "/usr/bin/ast-grep",
926    ];
927    for candidate in candidates {
928        let path = PathBuf::from(candidate);
929        if path.exists() {
930            return Ok(path);
931        }
932    }
933    // Fall back to PATH lookup via `which`-style check.
934    which_ast_grep()
935}
936
937fn which_ast_grep() -> Result<PathBuf, ExtractorError> {
938    // Use `which` on Unix.
939    let output = Command::new("which")
940        .arg("ast-grep")
941        .output()
942        .map_err(|_| ExtractorError::BinaryNotFound)?;
943    if output.status.success() {
944        let path_str = String::from_utf8_lossy(&output.stdout).trim().to_string();
945        if !path_str.is_empty() {
946            return Ok(PathBuf::from(path_str));
947        }
948    }
949    Err(ExtractorError::BinaryNotFound)
950}
951
952// ── Tests ─────────────────────────────────────────────────────────────────────
953
954#[cfg(test)]
955mod tests {
956    use super::*;
957
958    // ── Unit tests: strip_comment_prefixes ──
959
960    #[test]
961    fn strip_double_slash_prefix() {
962        let result = strip_comment_prefixes("// @g8.capability(name = \"x\")");
963        assert!(result.contains("@g8.capability"), "got: {result}");
964    }
965
966    #[test]
967    fn strip_hash_prefix() {
968        let result = strip_comment_prefixes("# @g8.capability(name = \"x\")");
969        assert!(result.contains("@g8.capability"), "got: {result}");
970    }
971
972    #[test]
973    fn strip_multiline_comment() {
974        let text =
975            "// @g8.capability(name = \"http-fetch\",\n//                 status = \"in_flight\")";
976        let result = strip_comment_prefixes(text);
977        assert!(result.contains("@g8.capability"), "got: {result}");
978        assert!(result.contains("http-fetch"), "got: {result}");
979        assert!(result.contains("in_flight"), "got: {result}");
980    }
981
982    // ── Unit tests: classify_heading ──
983
984    #[test]
985    fn classify_architecture_heading() {
986        let (kind, is_owner, is_parked) = classify_heading("Architecture");
987        assert_eq!(kind, IntentKind::ArchitecturalScope);
988        assert!(!is_owner);
989        assert!(!is_parked);
990    }
991
992    #[test]
993    fn classify_boundaries_heading() {
994        let (kind, is_owner, is_parked) = classify_heading("Boundaries");
995        assert_eq!(kind, IntentKind::Boundary);
996        assert!(!is_owner);
997        assert!(!is_parked);
998    }
999
1000    #[test]
1001    fn classify_parked_ideas_heading() {
1002        let (_, _, is_parked) = classify_heading("Parked Ideas");
1003        assert!(is_parked);
1004    }
1005
1006    #[test]
1007    fn classify_owner_heading() {
1008        let (_, is_owner, _) = classify_heading("Owner");
1009        assert!(is_owner);
1010    }
1011
1012    #[test]
1013    fn classify_unknown_heading_is_unclassified() {
1014        let (kind, is_owner, is_parked) = classify_heading("Random Section");
1015        assert_eq!(kind, IntentKind::Unclassified);
1016        assert!(!is_owner);
1017        assert!(!is_parked);
1018    }
1019
1020    // ── Unit tests: is_tests_dir ──
1021
1022    #[test]
1023    fn tests_dir_detection() {
1024        assert!(is_tests_dir(Path::new("src/tests/foo.rs")));
1025        assert!(is_tests_dir(Path::new(
1026            "crates/core/tests-integration/bar.rs"
1027        )));
1028        assert!(is_tests_dir(Path::new("unit-tests/baz.rs")));
1029        assert!(!is_tests_dir(Path::new("src/lib.rs")));
1030        assert!(!is_tests_dir(Path::new("src/test_helper.rs")));
1031    }
1032
1033    // ── Unit tests: parse_parked_line ──
1034
1035    #[test]
1036    fn parse_parked_line_with_reason() {
1037        let (name, reason) = parse_parked_line("config-hot-reload: requires Arc<RwLock<Config>>");
1038        assert_eq!(name, "config-hot-reload");
1039        assert_eq!(reason.unwrap(), "requires Arc<RwLock<Config>>");
1040    }
1041
1042    #[test]
1043    fn parse_parked_line_without_reason() {
1044        let (name, reason) = parse_parked_line("ml-scoring");
1045        assert_eq!(name, "ml-scoring");
1046        assert!(reason.is_none());
1047    }
1048
1049    // ── Unit test: RawAstMatch JSON deserialization ──
1050
1051    #[test]
1052    fn raw_ast_match_deserializes_from_real_ast_grep_output() {
1053        // Exact format emitted by ast-grep scan --json=stream
1054        let line = r#"{"text":"// @g8.capability(name = \"test-only-cap\", status = \"in_flight\", dev_only = true)","range":{"byteOffset":{"start":163,"end":244},"start":{"line":4,"column":0},"end":{"line":4,"column":81}},"file":"/tmp/test.rs","lines":"// @g8.capability(name = \"test-only-cap\", status = \"in_flight\", dev_only = true)","charCount":{"leading":0,"trailing":0},"language":"Rust","ruleId":"g8-rust-magic-comment","severity":"hint","note":null,"message":"G8 magic-comment annotation (Rust)","labels":[]}"#;
1055        let result: Result<RawAstMatch, _> = serde_json::from_str(line);
1056        match &result {
1057            Err(e) => panic!("JSON parse failed: {e}\nLine: {line}"),
1058            Ok(m) => {
1059                assert_eq!(m.text, "// @g8.capability(name = \"test-only-cap\", status = \"in_flight\", dev_only = true)");
1060                assert_eq!(m.range.start.line, 4);
1061                assert_eq!(m.range.start.column, 0);
1062                assert_eq!(m.file, "/tmp/test.rs");
1063                assert_eq!(m.rule_id.as_deref(), Some("g8-rust-magic-comment"));
1064            }
1065        }
1066    }
1067
1068    #[test]
1069    fn convert_ast_match_roundtrip() {
1070        // Verifies that a real ast-grep JSON line round-trips to a valid Annotation
1071        // including conditional=true when dev_only=true is present.
1072        let line = r#"{"text":"// @g8.capability(name = \"test-only-cap\", status = \"in_flight\", dev_only = true)","range":{"byteOffset":{"start":163,"end":244},"start":{"line":4,"column":0},"end":{"line":4,"column":81}},"file":"/tmp/test.rs","lines":"// @g8.capability(name = \"test-only-cap\", status = \"in_flight\", dev_only = true)","charCount":{"leading":0,"trailing":0},"language":"Rust","ruleId":"g8-rust-magic-comment","severity":"hint","note":null,"message":"G8 magic-comment annotation (Rust)","labels":[]}"#;
1073        let raw: RawAstMatch = serde_json::from_str(line).expect("parse");
1074        let name_re = Regex::new(r#"name\s*=\s*"([^"]+)""#).unwrap();
1075        let result = convert_ast_match(&raw, &name_re);
1076        match result {
1077            Err(e) => panic!("convert_ast_match failed: {e}"),
1078            Ok(None) => panic!("convert_ast_match returned None unexpectedly"),
1079            Ok(Some(ann)) => {
1080                assert_eq!(ann.kind, AnnotationKind::Capability);
1081                assert_eq!(ann.name.as_deref(), Some("test-only-cap"));
1082                assert!(ann.conditional, "dev_only=true should set conditional");
1083            }
1084        }
1085    }
1086
1087    // ── Unit tests: is_conditional ──
1088
1089    #[test]
1090    fn dev_only_sets_conditional() {
1091        let mut fields = BTreeMap::new();
1092        fields.insert("dev_only".to_string(), AnnotationValue::Bool(true));
1093        assert!(is_conditional("", &fields));
1094    }
1095
1096    #[test]
1097    fn cfg_attr_sets_conditional() {
1098        let fields = BTreeMap::new();
1099        assert!(is_conditional("#[cfg_attr(test, g8(...))]", &fields));
1100    }
1101
1102    // ── Embedded rules materialization ──
1103
1104    #[test]
1105    fn embedded_rules_materialize_deterministically() {
1106        let first = materialize_embedded_rules().unwrap();
1107        let second = materialize_embedded_rules().unwrap();
1108        assert_eq!(first, second);
1109        assert_eq!(std::fs::read_to_string(&first).unwrap(), EMBEDDED_RULES);
1110    }
1111
1112    // ── Integration tests: parse_sidecar ──
1113
1114    #[test]
1115    fn parse_agents_md_fixture() {
1116        let fixture = Path::new(concat!(
1117            env!("CARGO_MANIFEST_DIR"),
1118            "/tests/fixtures/markdown/AGENTS.md"
1119        ));
1120        if !fixture.exists() {
1121            return; // fixture not created yet (but it should be)
1122        }
1123        let annotations = parse_sidecar(fixture, IntentSourceKind::AgentsMd).unwrap();
1124        assert!(
1125            !annotations.is_empty(),
1126            "expected annotations from AGENTS.md fixture"
1127        );
1128        // At least one ArchitecturalScope intent.
1129        let has_arch = annotations.iter().any(|a| {
1130            a.fields
1131                .get("intent_kind")
1132                .and_then(|v| {
1133                    if let AnnotationValue::String(s) = v {
1134                        Some(s.as_str())
1135                    } else {
1136                        None
1137                    }
1138                })
1139                .map(|s| s == "architectural_scope")
1140                .unwrap_or(false)
1141        });
1142        assert!(has_arch, "expected at least one architectural_scope intent");
1143    }
1144}