amql_engine/extractor/

runner.rs

//! Subprocess execution and JSON parsing for extractors.

use crate::error::AqlError;
use crate::manifest::{ExtractorConfig, Manifest};
use crate::store::Annotation;
use crate::types::{AttrName, Binding, ProjectRoot, RelativePath, TagName};
use rustc_hash::{FxHashMap, FxHashSet};
use serde::{Deserialize, Serialize};
use serde_json::Value as JsonValue;
use std::process::Command;
use std::time::Duration;

/// Maximum time an extractor subprocess may run before being killed.
const EXTRACTOR_TIMEOUT: Duration = Duration::from_secs(30);

/// Result of running a single extractor.
#[derive(Debug, Clone, Serialize)]
pub struct ExtractorResult {
    /// Annotations discovered by the extractor.
    pub annotations: Vec<Annotation>,
    /// Non-fatal errors encountered during extraction.
    pub errors: Vec<String>,
}

/// JSON schema for a single annotation emitted by an extractor.
#[derive(Debug, Deserialize)]
struct RawExtractorAnnotation {
    file: String,
    bind: String,
    tag: String,
    #[serde(default)]
    attrs: FxHashMap<String, JsonValue>,
    #[serde(default)]
    children: Vec<RawExtractorAnnotation>,
}

/// Top-level JSON output from an extractor process.
#[derive(Debug, Deserialize)]
struct RawExtractorOutput {
    annotations: Vec<RawExtractorAnnotation>,
}

/// Recursively convert a raw extractor annotation into an Annotation.
fn convert_raw_annotation(raw: RawExtractorAnnotation) -> Annotation {
    let attrs: FxHashMap<AttrName, JsonValue> = raw
        .attrs
        .into_iter()
        .map(|(k, v)| (AttrName::from(k), v))
        .collect();

    let children: Vec<Annotation> = raw
        .children
        .into_iter()
        .map(convert_raw_annotation)
        .collect();

    Annotation {
        tag: TagName::from(raw.tag),
        attrs,
        binding: Binding::from(raw.bind),
        file: RelativePath::from(raw.file),
        children,
    }
}

/// Run a single extractor as a subprocess and parse its JSON output.
///
/// The extractor command is executed via `sh -c` with the project root as
/// the working directory. Stdout must be valid JSON matching the extractor
/// output schema. Stderr is captured but ignored on success.
pub fn run_extractor(
    config: &ExtractorConfig,
    project_root: &ProjectRoot,
) -> Result<ExtractorResult, AqlError> {
    let mut child = Command::new("sh")
        .arg("-c")
        .arg(&config.run)
        .current_dir(project_root.as_ref())
        .stdout(std::process::Stdio::piped())
        .stderr(std::process::Stdio::piped())
        .spawn()
        .map_err(|e| format!("Failed to execute extractor '{}': {e}", config.name))?;

    // Read stdout and stderr on separate threads to avoid pipe buffer
    // deadlock. If the child produces more output than the OS pipe buffer
    // (~64 KB), it will block on write. Draining concurrently prevents this.
    let stdout_handle = child.stdout.take();
    let stderr_handle = child.stderr.take();

    let stdout_thread = std::thread::spawn(move || -> Vec<u8> {
        use std::io::Read;
        let mut buf = Vec::new();
        if let Some(mut stdout) = stdout_handle {
            let _ = stdout.read_to_end(&mut buf);
        }
        buf
    });

    let stderr_thread = std::thread::spawn(move || -> Vec<u8> {
        use std::io::Read;
        let mut buf = Vec::new();
        if let Some(mut stderr) = stderr_handle {
            let _ = stderr.read_to_end(&mut buf);
        }
        buf
    });

    let deadline = std::time::Instant::now() + EXTRACTOR_TIMEOUT;
    loop {
        match child.try_wait() {
            Ok(Some(_)) => break,
            Ok(None) => {
                if std::time::Instant::now() >= deadline {
                    let _ = child.kill();
                    let _ = child.wait();
                    return Err(format!(
                        "Extractor '{}' timed out after {}s",
                        config.name,
                        EXTRACTOR_TIMEOUT.as_secs()
                    )
                    .into());
                }
                std::thread::sleep(Duration::from_millis(50));
            }
            Err(e) => {
                return Err(format!("Failed to wait on extractor '{}': {e}", config.name).into());
            }
        }
    }

    let status = child
        .wait()
        .map_err(|e| format!("Failed to wait on extractor '{}': {e}", config.name))?;

    let stdout_bytes = stdout_thread.join().unwrap_or_default();
    let stderr_bytes = stderr_thread.join().unwrap_or_default();

    let output = std::process::Output {
        status,
        stdout: stdout_bytes,
        stderr: stderr_bytes,
    };

    if !output.status.success() {
        let stderr = String::from_utf8_lossy(&output.stderr);
        return Err(format!(
            "Extractor '{}' exited with {}: {}",
            config.name,
            output.status,
            stderr.trim()
        )
        .into());
    }

    let stdout = String::from_utf8_lossy(&output.stdout);
    let raw: RawExtractorOutput = serde_json::from_str(&stdout)
        .map_err(|e| format!("Extractor '{}' produced invalid JSON: {e}", config.name))?;

    let mut annotations = Vec::with_capacity(raw.annotations.len());
    let mut errors = Vec::new();

    for entry in raw.annotations {
        if entry.file.is_empty() {
            errors.push(format!(
                "Extractor '{}': annotation missing 'file' field",
                config.name
            ));
            continue;
        }
        if entry.bind.is_empty() {
            errors.push(format!(
                "Extractor '{}': annotation for '{}' missing 'bind' field",
                config.name, entry.file
            ));
            continue;
        }

        annotations.push(convert_raw_annotation(entry));
    }

    Ok(ExtractorResult {
        annotations,
        errors,
    })
}

/// Run all extractors defined in the manifest and collect results.
///
/// For each manifest extractor: if `run` is non-empty, execute as a
/// subprocess. If `run` is empty, look up a matching built-in by name
/// in the registry. Extractors that fail are logged as errors in their
/// result rather than halting the entire pipeline.
#[must_use = "extractor results contain annotations to merge into the store"]
pub fn run_all_extractors(
    manifest: &Manifest,
    project_root: &ProjectRoot,
    registry: &super::ExtractorRegistry,
) -> Vec<ExtractorResult> {
    manifest
        .extractors
        .iter()
        .map(|config| {
            if config.run.is_empty() {
                // Try built-in extractor
                match registry.get(&config.name) {
                    Some(builtin) => run_builtin(builtin, config, project_root),
                    None => ExtractorResult {
                        annotations: vec![],
                        errors: vec![format!(
                            "Extractor '{}': no 'run' command and no built-in found",
                            config.name
                        )],
                    },
                }
            } else {
                match run_extractor(config, project_root) {
                    Ok(result) => result,
                    Err(e) => ExtractorResult {
                        annotations: vec![],
                        errors: vec![e.to_string()],
                    },
                }
            }
        })
        .collect()
}

/// Run a built-in extractor against files matching the config's globs.
fn run_builtin(
    builtin: &dyn super::BuiltinExtractor,
    config: &ExtractorConfig,
    project_root: &ProjectRoot,
) -> ExtractorResult {
    let extensions: FxHashSet<String> = builtin
        .extensions()
        .iter()
        .map(|ext| ext.trim_start_matches('.').to_string())
        .collect();

    let mut annotations = Vec::new();
    let mut errors = Vec::new();

    let walker = ignore::WalkBuilder::new(project_root.as_ref())
        .hidden(false)
        .git_ignore(true)
        .git_global(true)
        .git_exclude(true)
        .filter_entry(|entry| {
            let name = entry.file_name().to_string_lossy();
            if entry.file_type().is_some_and(|ft| ft.is_dir()) {
                return !crate::paths::SKIP_DIRS.contains(&name.as_ref());
            }
            true
        })
        .build();

    for entry in walker {
        let entry = match entry {
            Ok(e) => e,
            Err(e) => {
                errors.push(format!("Extractor '{}': walk error: {e}", config.name));
                continue;
            }
        };

        let path = entry.path();
        if !path.is_file() {
            continue;
        }

        // Check extension
        let ext = path.extension().and_then(|e| e.to_str()).unwrap_or("");
        if !extensions.contains(ext) {
            continue;
        }

        // Skip declaration files
        let name = path.file_name().and_then(|n| n.to_str()).unwrap_or("");
        if name.ends_with(".d.ts") || name.ends_with(".d.mts") || name.ends_with(".d.cts") {
            continue;
        }

        // Apply glob filter if specified
        if !config.globs.is_empty() {
            let relative_str = path
                .strip_prefix(project_root.as_ref())
                .ok()
                .and_then(|r| r.to_str())
                .unwrap_or("");
            let matches_glob = config.globs.iter().any(|g| glob_matches(g, relative_str));
            if !matches_glob {
                continue;
            }
        }

        let relative = match path.strip_prefix(project_root.as_ref()) {
            Ok(r) => RelativePath::from(r.to_string_lossy().as_ref()),
            Err(_) => continue,
        };

        let source = match std::fs::read_to_string(path) {
            Ok(s) => s,
            Err(e) => {
                errors.push(format!(
                    "Extractor '{}': failed to read {}: {e}",
                    config.name,
                    path.display()
                ));
                continue;
            }
        };

        let file_anns = builtin.extract(&source, &relative);
        annotations.extend(file_anns);
    }

    ExtractorResult {
        annotations,
        errors,
    }
}

/// Simple glob matching: supports `*` and `**` prefix patterns.
fn glob_matches(pattern: &str, path: &str) -> bool {
    if pattern.starts_with("**") {
        // **/*.ts matches any path ending with the suffix
        let suffix = pattern.trim_start_matches("**/").trim_start_matches("*/");
        if suffix.starts_with("*.") {
            let ext = &suffix[1..]; // e.g. ".ts"
            return path.ends_with(ext);
        }
        path.contains(suffix)
    } else if let Some(idx) = pattern.find('*') {
        let prefix = &pattern[..idx];
        let suffix = &pattern[idx + 1..];
        path.starts_with(prefix) && path.ends_with(suffix)
    } else {
        path == pattern
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::path::PathBuf;

    fn test_project_root() -> ProjectRoot {
        ProjectRoot::from(PathBuf::from("/tmp"))
    }

    #[test]
    fn parses_valid_extractor_json() {
        // Arrange
        let json = r#"{"annotations":[
            {"file":"src/api.ts","bind":"createUser","tag":"controller","attrs":{"method":"POST","path":"/api/users"}},
            {"file":"src/api.ts","bind":"getUser","tag":"controller","attrs":{"method":"GET","path":"/api/users/:id"}}
        ]}"#;
        let raw: RawExtractorOutput = serde_json::from_str(json).unwrap();

        // Act
        let annotations: Vec<Annotation> = raw
            .annotations
            .into_iter()
            .map(convert_raw_annotation)
            .collect();

        // Assert
        assert_eq!(annotations.len(), 2, "should parse two annotations");
        assert_eq!(
            annotations[0].tag, "controller",
            "first tag should be controller"
        );
        assert_eq!(
            annotations[0].binding, "createUser",
            "first binding should be createUser"
        );
        assert_eq!(
            annotations[0].attrs.get("method"),
            Some(&JsonValue::String("POST".to_string())),
            "first method should be POST"
        );
        assert_eq!(
            annotations[1].binding, "getUser",
            "second binding should be getUser"
        );
    }

    #[test]
    fn parses_nested_children() {
        // Arrange
        let json = r#"{"annotations":[
            {"file":"src/app.test.ts","bind":"AppTests","tag":"describe","attrs":{},
             "children":[
                {"file":"src/app.test.ts","bind":"renders correctly","tag":"test","attrs":{}},
                {"file":"src/app.test.ts","bind":"nested suite","tag":"describe","attrs":{},
                 "children":[
                    {"file":"src/app.test.ts","bind":"handles edge case","tag":"test","attrs":{}}
                 ]}
             ]}
        ]}"#;
        let raw: RawExtractorOutput = serde_json::from_str(json).unwrap();

        // Act
        let annotations: Vec<Annotation> = raw
            .annotations
            .into_iter()
            .map(convert_raw_annotation)
            .collect();

        // Assert
        assert_eq!(
            annotations.len(),
            1,
            "should parse one top-level annotation"
        );
        assert_eq!(
            annotations[0].tag, "describe",
            "top-level should be describe"
        );
        assert_eq!(
            annotations[0].children.len(),
            2,
            "describe should have 2 children"
        );
        assert_eq!(
            annotations[0].children[0].tag, "test",
            "first child should be test"
        );
        assert_eq!(
            annotations[0].children[1].tag, "describe",
            "second child should be describe"
        );
        assert_eq!(
            annotations[0].children[1].children.len(),
            1,
            "nested describe should have 1 child"
        );
        assert_eq!(
            annotations[0].children[1].children[0].tag, "test",
            "deeply nested should be test"
        );
    }

    #[test]
    fn skips_entries_with_missing_fields() {
        // Arrange
        let json = r#"{"annotations":[
            {"file":"","bind":"x","tag":"t","attrs":{}},
            {"file":"f.ts","bind":"","tag":"t","attrs":{}},
            {"file":"f.ts","bind":"ok","tag":"t","attrs":{}}
        ]}"#;
        let raw: RawExtractorOutput = serde_json::from_str(json).unwrap();
        let config_name = "test";

        // Act
        let mut annotations = Vec::new();
        let mut errors = Vec::new();
        for entry in raw.annotations {
            if entry.file.is_empty() {
                errors.push(format!(
                    "Extractor '{}': annotation missing 'file' field",
                    config_name
                ));
                continue;
            }
            if entry.bind.is_empty() {
                errors.push(format!(
                    "Extractor '{}': annotation for '{}' missing 'bind' field",
                    config_name, entry.file
                ));
                continue;
            }
            annotations.push(convert_raw_annotation(entry));
        }

        // Assert
        assert_eq!(annotations.len(), 1, "should keep only valid annotation");
        assert_eq!(errors.len(), 2, "should report two errors");
    }

    #[test]
    fn run_extractor_with_echo() {
        // Arrange
        let config = ExtractorConfig {
            name: "echo-test".to_string(),
            run: r#"echo '{"annotations":[{"file":"src/app.ts","bind":"handler","tag":"endpoint","attrs":{"method":"GET"}}]}'"#.to_string(),
            globs: vec!["src/**/*.ts".to_string()],
        };

        // Act
        let result = run_extractor(&config, &test_project_root()).unwrap();

        // Assert
        assert_eq!(result.annotations.len(), 1, "should produce one annotation");
        assert_eq!(result.errors.len(), 0, "should produce no errors");
        assert_eq!(
            result.annotations[0].tag, "endpoint",
            "tag should be endpoint"
        );
        assert_eq!(
            result.annotations[0].binding, "handler",
            "binding should be handler"
        );
    }

    #[test]
    fn run_extractor_handles_failure() {
        // Arrange
        let config = ExtractorConfig {
            name: "fail-test".to_string(),
            run: "exit 1".to_string(),
            globs: vec![],
        };

        // Act
        let result = run_extractor(&config, &test_project_root());

        // Assert
        assert!(result.is_err(), "should return error for failed extractor");
        assert!(
            result.unwrap_err().to_string().contains("fail-test"),
            "error should mention extractor name"
        );
    }

    #[test]
    fn run_extractor_handles_invalid_json() {
        // Arrange
        let config = ExtractorConfig {
            name: "bad-json".to_string(),
            run: "echo 'not json'".to_string(),
            globs: vec![],
        };

        // Act
        let result = run_extractor(&config, &test_project_root());

        // Assert
        assert!(result.is_err(), "should return error for invalid JSON");
        assert!(
            result.unwrap_err().to_string().contains("invalid JSON"),
            "error should mention invalid JSON"
        );
    }

    #[test]
    fn run_all_extractors_continues_on_failure() {
        // Arrange
        let manifest = Manifest {
            version: "1.0".to_string(),
            tags: FxHashMap::default(),
            audiences: FxHashMap::default(),
            visibilities: FxHashMap::default(),
            extractors: vec![
                ExtractorConfig {
                    name: "failing".to_string(),
                    run: "exit 1".to_string(),
                    globs: vec![],
                },
                ExtractorConfig {
                    name: "passing".to_string(),
                    run: r#"echo '{"annotations":[{"file":"a.ts","bind":"b","tag":"t","attrs":{}}]}'"#.to_string(),
                    globs: vec![],
                },
            ],
        };

        // Act
        let registry = crate::extractor::ExtractorRegistry::new();
        let results = run_all_extractors(&manifest, &test_project_root(), &registry);

        // Assert
        assert_eq!(results.len(), 2, "should return result for each extractor");
        assert!(!results[0].errors.is_empty(), "first should have errors");
        assert_eq!(
            results[1].annotations.len(),
            1,
            "second should have annotations"
        );
    }
}