meta_ast/extractor/

mod.rs

//! Parallel file extraction orchestration.
//!
//! Uses rayon `par_iter` to read, parse, and extract symbols/imports/
//! references across files concurrently. Each file is processed
//! independently; errors are accumulated as diagnostics per file.
//!
//! Set `ExtractOptions::skip_imports_and_refs` to `true` when only
//! symbol listing is needed (e.g. inspect mode); skips the import and
//! reference query passes, roughly halving per-file extraction time.
//!
//! Set `ExtractOptions::keep_text` to `true` to retain the analyzed source text
//! on each extraction. An editor or language server needs it to convert ranges
//! against the exact bytes the analysis read, instead of re-reading a file that
//! may have changed. Off by default: the CLI does not need it and the text is
//! held for the lifetime of every extraction.

use std::collections::HashMap;
use std::path::{Path, PathBuf};

use rayon::prelude::*;

use crate::error::{Diagnostic, Severity};
use crate::language::LangId;
use crate::model::{IdGenerator, Symbol, SymbolId};
use crate::parser;

pub use crate::model::FileExtraction;

/// Largest source file the extractor reads, in bytes.
///
/// A generated file above this size is reported and skipped: parsing it would
/// spike memory for a result no consumer can use, and the diagnostic keeps the
/// skip visible instead of silently dropping the file.
pub const MAX_SOURCE_BYTES: u64 = 8 * 1024 * 1024;

/// Controls what the extraction pass produces.
#[derive(Debug, Clone, Copy, Default)]
pub struct ExtractOptions {
    /// Skip import and reference extraction entirely; only extract symbols
    /// and AST node counts. Halves per-file extraction cost for pure
    /// symbol-inspection workflows.
    pub skip_imports_and_refs: bool,
    /// Retain the analyzed source text on each extraction. Off by default:
    /// the CLI does not need it, and an editor or language server uses it to
    /// convert ranges against the exact text the analysis read.
    pub keep_text: bool,
}

pub struct ExtractionResult {
    pub files: Vec<FileExtraction>,
}

/// ID allocation state shared by disk and in-memory extraction.
#[derive(Debug, Default)]
pub struct ExtractionIdGenerators {
    symbols: IdGenerator<SymbolId>,
    #[cfg(feature = "dataflow")]
    data_nodes: IdGenerator<crate::model::DataNodeId>,
}

impl ExtractionIdGenerators {
    pub fn new() -> Self {
        Self::default()
    }

    pub fn with_symbol_start(symbol_start: u32) -> Self {
        Self {
            symbols: IdGenerator::with_start(symbol_start),
            #[cfg(feature = "dataflow")]
            data_nodes: IdGenerator::new(),
        }
    }

    #[cfg(feature = "dataflow")]
    pub fn with_starts(symbol_start: u32, data_node_start: u32) -> Self {
        Self {
            symbols: IdGenerator::with_start(symbol_start),
            data_nodes: IdGenerator::with_start(data_node_start),
        }
    }

    pub fn symbols(&self) -> &IdGenerator<SymbolId> {
        &self.symbols
    }

    #[cfg(feature = "dataflow")]
    pub fn data_nodes(&self) -> &IdGenerator<crate::model::DataNodeId> {
        &self.data_nodes
    }
}

/// Source text supplied by an editor buffer.
#[derive(Debug, Clone, Copy)]
pub struct InMemorySource<'a> {
    pub uri: &'a str,
    pub text: &'a str,
    pub version: i32,
    pub language: LangId,
}

/// Extraction result tied to the editor document version that produced it.
#[derive(Debug, Clone)]
pub struct VersionedExtraction {
    pub uri: String,
    pub version: i32,
    pub file: FileExtraction,
}

pub fn extract(files: &[(std::path::PathBuf, LangId)]) -> ExtractionResult {
    extract_with_options(files, &ExtractOptions::default())
}

pub fn extract_with_options(
    files: &[(std::path::PathBuf, LangId)],
    opts: &ExtractOptions,
) -> ExtractionResult {
    let id_generators = ExtractionIdGenerators::new();
    extract_with_id_gen(files, opts, &id_generators)
}

pub fn extract_with_id_gen(
    files: &[(PathBuf, LangId)],
    opts: &ExtractOptions,
    id_generators: &ExtractionIdGenerators,
) -> ExtractionResult {
    // Phase one parses in parallel with file-local numbering, so no atomic
    // ordering reaches the output. Phase two numbers the files in path order.
    let mut file_extractions: Vec<FileExtraction> = files
        .par_iter()
        .map(|(path, lang)| {
            let local = ExtractionIdGenerators::new();
            extract_single_file(path, lang, &local, opts)
        })
        .collect();

    file_extractions.sort_by(|a, b| a.path.cmp(&b.path));

    for file in &mut file_extractions {
        renumber(file, id_generators);
    }

    ExtractionResult {
        files: file_extractions,
    }
}

/// Map the file-local identifiers onto the shared generator.
///
/// The new numbering follows the symbol and data node order inside the file,
/// which is deterministic for identical input.
fn renumber(file: &mut FileExtraction, id_generators: &ExtractionIdGenerators) {
    let symbol_map = if file.symbols.is_empty() {
        HashMap::new()
    } else {
        let counter = IdGenerator::<SymbolId>::with_start(
            id_generators.symbols.reserve(file.symbols.len() as u32),
        );
        let mut map = HashMap::with_capacity(file.symbols.len());
        for symbol in &mut file.symbols {
            let next = counter.next();
            map.insert(symbol.id.to_raw(), next.to_raw());
            symbol.id = next;
        }
        map
    };

    #[cfg(feature = "dataflow")]
    renumber_data_nodes(file, id_generators, &symbol_map);

    #[cfg(not(feature = "dataflow"))]
    drop(symbol_map);
}

#[cfg(feature = "dataflow")]
fn renumber_data_nodes(
    file: &mut FileExtraction,
    id_generators: &ExtractionIdGenerators,
    symbol_map: &HashMap<u32, u32>,
) {
    use crate::model::DataNodeId;

    if file.data_nodes.is_empty() {
        file.flow_edges.clear();
        return;
    }

    let counter = IdGenerator::<DataNodeId>::with_start(
        id_generators
            .data_nodes
            .reserve(file.data_nodes.len() as u32),
    );
    let mut node_map = HashMap::with_capacity(file.data_nodes.len());
    for node in &mut file.data_nodes {
        let next = counter.next();
        node_map.insert(node.id.to_raw(), next.to_raw());
        node.id = next;
        node.symbol_id = node
            .symbol_id
            .and_then(|id| symbol_map.get(&id.to_raw()).copied())
            .and_then(SymbolId::new);
    }

    let node_ids: HashMap<u32, DataNodeId> = node_map
        .iter()
        .filter_map(|(&old, &new)| Some((old, DataNodeId::new(new)?)))
        .collect();
    file.flow_edges.retain_mut(|edge| {
        let (Some(source), Some(target)) = (
            node_ids.get(&edge.source.to_raw()).copied(),
            node_ids.get(&edge.target.to_raw()).copied(),
        ) else {
            return false;
        };
        edge.source = source;
        edge.target = target;
        true
    });
}

fn extract_single_file(
    path: &Path,
    lang: &LangId,
    id_generators: &ExtractionIdGenerators,
    opts: &ExtractOptions,
) -> FileExtraction {
    match std::fs::metadata(path) {
        Ok(metadata) if metadata.len() > MAX_SOURCE_BYTES => {
            return failed_extraction(
                path,
                *lang,
                format!(
                    "file is {} bytes, over the {MAX_SOURCE_BYTES} byte limit for a single source",
                    metadata.len()
                ),
            );
        }
        Ok(_) => {}
        Err(error) => {
            return failed_extraction(path, *lang, format!("failed to read file: {error}"));
        }
    }

    let source = match std::fs::read(path) {
        Ok(source) => source,
        Err(error) => {
            return failed_extraction(path, *lang, format!("failed to read file: {error}"));
        }
    };

    extract_source(path, *lang, &source, id_generators, opts)
}

/// Extract an open editor buffer without reading its backing file.
pub fn extract_text_with_id_gen(
    source: InMemorySource<'_>,
    opts: &ExtractOptions,
    id_generators: &ExtractionIdGenerators,
) -> Result<VersionedExtraction, crate::Error> {
    let parsed_uri =
        url::Url::parse(source.uri).map_err(|error| crate::Error::InvalidSourceUri {
            uri: source.uri.to_string(),
            message: error.to_string(),
        })?;
    let path = parsed_uri
        .to_file_path()
        .map_err(|()| crate::Error::InvalidSourceUri {
            uri: source.uri.to_string(),
            message: "URI must use the file scheme and contain an absolute path".to_string(),
        })?;
    let path = crate::input::simplified_path(&path).to_path_buf();
    let file = extract_source(
        &path,
        source.language,
        source.text.as_bytes(),
        id_generators,
        opts,
    );

    Ok(VersionedExtraction {
        uri: source.uri.to_string(),
        version: source.version,
        file,
    })
}

fn extract_source(
    path: &Path,
    lang: LangId,
    source: &[u8],
    id_generators: &ExtractionIdGenerators,
    opts: &ExtractOptions,
) -> FileExtraction {
    let tree = match crate::parser::parse_tree(lang, source) {
        Ok(t) => t,
        Err(e) => {
            return failed_extraction(path, lang, e.to_string());
        }
    };

    let metrics = parser::tree_metrics(&tree, source);
    let mut diags = Vec::new();

    if tree.root_node().has_error() {
        diags.push(Diagnostic {
            path: path.to_path_buf(),
            severity: Severity::Warning,
            message: format!(
                "file has {:.0}% parse errors, results may be incomplete",
                metrics.error_ratio * 100.0
            ),
            source_range: None,
        });
    }

    let raw_symbols = match crate::language::extract_symbols_for_checked(lang, &tree, source) {
        Ok(symbols) => symbols,
        Err(error) => {
            diags.push(Diagnostic {
                path: path.to_path_buf(),
                severity: Severity::Error,
                message: format!("symbol extraction failed: {error}"),
                source_range: None,
            });
            Vec::new()
        }
    };
    let symbols = raw_symbols
        .into_iter()
        .map(|raw| Symbol {
            id: id_generators.symbols.next(),
            name: raw.name.into_owned(),
            kind: raw.kind,
            language: lang,
            file_path: path.to_path_buf(),
            source_range: raw.source_range,
            name_range: raw.name_range,
            visibility: raw.visibility,
            signature: raw.signature.map(|s| s.into_owned()),
            docstring: raw.docstring.map(|s| s.into_owned()),
            is_async: raw.is_async,
        })
        .collect();

    let (imports, references, text_diagnostics) = if opts.skip_imports_and_refs {
        (Vec::new(), Vec::new(), Vec::new())
    } else {
        match crate::language::extract_imports_and_references_for_checked(lang, &tree, source, path)
        {
            Ok(extracted) => extracted,
            Err(error) => {
                diags.push(Diagnostic {
                    path: path.to_path_buf(),
                    severity: Severity::Error,
                    message: format!("import and reference extraction failed: {error}"),
                    source_range: None,
                });
                (Vec::new(), Vec::new(), Vec::new())
            }
        }
    };

    #[cfg(feature = "metacall-deploy")]
    let call_sites = match crate::deploy::scanner::scan_file(lang, &tree, source, path) {
        Ok(sites) => sites,
        Err(error) => {
            diags.push(Diagnostic {
                path: path.to_path_buf(),
                severity: Severity::Error,
                message: format!("deploy call site scan failed: {error}"),
                source_range: None,
            });
            Vec::new()
        }
    };

    #[cfg(feature = "dataflow")]
    let (data_nodes, flow_edges) =
        crate::language::dataflow::extract_dataflow(lang, &tree, source, &id_generators.data_nodes);

    let mut out = FileExtraction::empty(path.to_path_buf(), lang);
    out.symbols = symbols;
    out.imports = imports;
    out.references = references;
    out.diagnostics = diags;
    out.diagnostics.extend(text_diagnostics);
    out.ast_node_count = metrics.node_count;
    // Ranges index the analyzed bytes, so a lossy copy would shift every
    // offset. A file that is not valid UTF-8 keeps no text and the consumer
    // falls back to its own source lookup.
    if opts.keep_text {
        out.text = std::str::from_utf8(source)
            .ok()
            .map(std::sync::Arc::<str>::from);
    }
    #[cfg(feature = "metacall-deploy")]
    {
        out.call_sites = call_sites;
    }
    #[cfg(feature = "dataflow")]
    {
        out.data_nodes = data_nodes;
        out.flow_edges = flow_edges;
    }
    out
}

fn failed_extraction(path: &Path, lang: LangId, message: String) -> FileExtraction {
    FileExtraction::failed(path.to_path_buf(), lang, message)
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::path::PathBuf;

    fn test_dir() -> PathBuf {
        let dir = std::env::temp_dir().join("meta_ast_test_extractor");
        let _ = std::fs::create_dir_all(&dir);
        dir
    }

    fn write_temp(name: &str, content: &[u8]) -> PathBuf {
        let path = test_dir().join(name);
        std::fs::write(&path, content).unwrap();
        path
    }

    #[test]
    fn symbol_ids_are_contiguous_in_path_order() {
        let dir = test_dir().join("numbering");
        let _ = std::fs::remove_dir_all(&dir);
        std::fs::create_dir_all(&dir).unwrap();

        let mut files = Vec::new();
        for index in (0..24).rev() {
            let path = dir.join(format!("f{index:02}.py"));
            let body: String = (0..5)
                .map(|n| format!("def g{index}_{n}(): pass\n"))
                .collect();
            std::fs::write(&path, body).unwrap();
            files.push((path, LangId::Python));
        }

        let result = extract(&files);
        let mut per_file: Vec<(PathBuf, Vec<u32>)> = result
            .files
            .iter()
            .map(|file| {
                (
                    file.path.clone(),
                    file.symbols.iter().map(|s| s.id.to_raw()).collect(),
                )
            })
            .collect();
        per_file.sort_by(|a, b| a.0.cmp(&b.0));

        let mut next = 1u32;
        for (path, ids) in &per_file {
            assert_eq!(ids.len(), 5, "{}", path.display());
            assert_eq!(
                ids.first().copied(),
                Some(next),
                "ids of {} must start at {next} in path order",
                path.display()
            );
            let last = ids.last().copied().unwrap();
            assert_eq!(
                last,
                next + ids.len() as u32 - 1,
                "ids of {} must be contiguous",
                path.display()
            );
            next = last + 1;
        }
    }

    #[test]
    fn symbol_ids_are_stable_across_runs() {
        let dir = test_dir().join("stable_numbering");
        let _ = std::fs::remove_dir_all(&dir);
        std::fs::create_dir_all(&dir).unwrap();

        let mut files = Vec::new();
        for index in 0..8 {
            let path = dir.join(format!("s{index}.py"));
            std::fs::write(
                &path,
                format!("def h{index}(): pass\ndef k{index}(): pass\n"),
            )
            .unwrap();
            files.push((path, LangId::Python));
        }

        let map = |result: ExtractionResult| {
            let mut entries: Vec<(String, u32)> = result
                .files
                .iter()
                .flat_map(|file| file.symbols.iter().map(|s| (s.name.clone(), s.id.to_raw())))
                .collect();
            entries.sort();
            entries
        };

        assert_eq!(map(extract(&files)), map(extract(&files)));
    }

    #[test]
    fn extract_single_python_file() {
        let path = write_temp("single.py", b"def hello(): pass\n");
        let result = extract(&[(path.clone(), LangId::Python)]);
        assert_eq!(result.files.len(), 1);
        assert!(!result.files[0].symbols.is_empty());
        assert!(result.files[0].diagnostics.is_empty());
        let names: Vec<&str> = result.files[0]
            .symbols
            .iter()
            .map(|s| s.name.as_str())
            .collect();
        assert!(names.contains(&"hello"));
    }

    #[test]
    fn extract_multiple_files_parallel() {
        let p1 = write_temp("file_a.py", b"def alpha(): pass\n");
        let p2 = write_temp("file_b.py", b"def beta(): pass\ndef gamma(): pass\n");
        let p3 = write_temp("file_c.py", b"class Delta: pass\n");

        let files = vec![
            (p1.clone(), LangId::Python),
            (p2.clone(), LangId::Python),
            (p3.clone(), LangId::Python),
        ];
        let result = extract(&files);
        let all_names: Vec<&str> = result
            .files
            .iter()
            .flat_map(|f| f.symbols.iter().map(|s| s.name.as_str()))
            .collect();
        assert!(all_names.contains(&"alpha"), "missing alpha: {all_names:?}");
        assert!(all_names.contains(&"beta"), "missing beta: {all_names:?}");
        assert!(all_names.contains(&"gamma"), "missing gamma: {all_names:?}");
        assert!(all_names.contains(&"Delta"), "missing Delta: {all_names:?}");
    }

    #[test]
    fn accumulate_diagnostics_on_malformed() {
        let path = test_dir().join("nonexistent_broken.py");
        let _ = std::fs::remove_file(&path);
        let result = extract(&[(path, LangId::Python)]);
        assert!(!result.files[0].diagnostics.is_empty());
    }

    #[test]
    fn partial_extraction_on_errors() {
        let valid = write_temp("valid_partial.py", b"def works(): pass\n");
        let broken = write_temp(
            "broken_partial.py",
            b"def broken(\n   # missing close paren and colon\n",
        );
        let result = extract(&[(valid.clone(), LangId::Python), (broken, LangId::Python)]);
        let names: Vec<&str> = result
            .files
            .iter()
            .flat_map(|f| f.symbols.iter().map(|s| s.name.as_str()))
            .collect();
        assert!(
            names.contains(&"works"),
            "valid file symbols should be present: {names:?}"
        );
    }

    #[test]
    fn output_deterministic() {
        let path = write_temp("deterministic.py", b"def foo(): pass\ndef bar(): pass\n");
        let files = vec![(path.clone(), LangId::Python)];

        let r1 = extract(&files);
        let r2 = extract(&files);

        let names1: Vec<String> = r1
            .files
            .iter()
            .flat_map(|f| f.symbols.iter().map(|s| s.name.clone()))
            .collect();
        let names2: Vec<String> = r2
            .files
            .iter()
            .flat_map(|f| f.symbols.iter().map(|s| s.name.clone()))
            .collect();
        assert_eq!(names1, names2);
    }

    #[test]
    fn in_memory_extraction_uses_unsaved_text_and_preserves_version() {
        let path = test_dir().join("buffer.py");
        let _ = std::fs::remove_file(&path);
        let uri = url::Url::from_file_path(&path).unwrap().to_string();
        let id_generators = ExtractionIdGenerators::with_symbol_start(40);

        let result = extract_text_with_id_gen(
            InMemorySource {
                uri: &uri,
                text: "def unsaved(): pass\n",
                version: 7,
                language: LangId::Python,
            },
            &ExtractOptions::default(),
            &id_generators,
        )
        .unwrap();

        assert_eq!(result.version, 7);
        assert_eq!(result.uri, uri);
        assert_eq!(result.file.path, path);
        assert_eq!(result.file.symbols[0].name, "unsaved");
        assert_eq!(result.file.symbols[0].id, SymbolId::new(40).unwrap());
    }

    #[test]
    fn text_retention_is_opt_in_for_both_entry_points() {
        let path = test_dir().join("retained.py");
        let source = "def retained(): pass\n";
        std::fs::write(&path, source).unwrap();
        let id_generators = ExtractionIdGenerators::new();
        let files = [(path.clone(), LangId::Python)];

        let without = extract_with_id_gen(&files, &ExtractOptions::default(), &id_generators);
        assert_eq!(without.files[0].text, None);

        let opts = ExtractOptions {
            keep_text: true,
            ..ExtractOptions::default()
        };
        let from_disk = extract_with_id_gen(&files, &opts, &id_generators);
        assert_eq!(from_disk.files[0].text.as_deref(), Some(source));

        let uri = url::Url::from_file_path(&path).unwrap().to_string();
        let from_buffer = extract_text_with_id_gen(
            InMemorySource {
                uri: &uri,
                text: source,
                version: 1,
                language: LangId::Python,
            },
            &opts,
            &id_generators,
        )
        .unwrap();
        assert_eq!(from_buffer.file.text.as_deref(), Some(source));
    }

    #[test]
    fn in_memory_extraction_rejects_non_file_uri() {
        let id_generators = ExtractionIdGenerators::new();
        let error = extract_text_with_id_gen(
            InMemorySource {
                uri: "untitled:buffer.py",
                text: "def value(): pass\n",
                version: 1,
                language: LangId::Python,
            },
            &ExtractOptions::default(),
            &id_generators,
        )
        .unwrap_err();

        assert!(matches!(error, crate::Error::InvalidSourceUri { .. }));
    }

    #[test]
    fn extraction_id_generators_accessors() {
        let id_generators = ExtractionIdGenerators::with_symbol_start(100);
        assert_eq!(id_generators.symbols().next(), SymbolId::new(100).unwrap());
        #[cfg(feature = "dataflow")]
        {
            let dual = ExtractionIdGenerators::with_starts(200, 300);
            assert_eq!(dual.symbols().next(), SymbolId::new(200).unwrap());
            assert_eq!(
                dual.data_nodes().next(),
                crate::model::DataNodeId::new(300).unwrap()
            );
        }
    }

    #[test]
    fn symbols_assigned_ids() {
        let path = write_temp("ids.py", b"def a(): pass\ndef b(): pass\ndef c(): pass\n");
        let result = extract(&[(path, LangId::Python)]);
        let ids: Vec<u32> = result.files[0]
            .symbols
            .iter()
            .map(|s| s.id.to_raw())
            .collect();
        let mut sorted_ids = ids.clone();
        sorted_ids.sort();
        assert_eq!(ids, sorted_ids, "IDs should be sequential");

        for window in sorted_ids.windows(2) {
            assert_eq!(window[1] - window[0], 1, "IDs should be consecutive");
        }

        let unique: std::collections::HashSet<u32> = ids.iter().copied().collect();
        assert_eq!(
            unique.len(),
            result.files[0].symbols.len(),
            "all IDs must be unique"
        );
    }

    #[test]
    fn parse_errors_are_reported_for_any_broken_tree() {
        let clean = write_temp("parse_clean.py", b"def ok(): pass\n");
        let slightly_broken = write_temp("parse_broken.py", b"def broken(\n   # no close paren\n");
        let heavily_broken = write_temp("parse_heavily_broken.py", b"@@@ ??? (((\n");
        let result = extract(&[
            (clean.clone(), LangId::Python),
            (slightly_broken.clone(), LangId::Python),
            (heavily_broken.clone(), LangId::Python),
        ]);

        let parse_diagnostics = |path: &PathBuf| -> usize {
            result
                .files
                .iter()
                .find(|f| &f.path == path)
                .unwrap()
                .diagnostics
                .iter()
                .filter(|d| d.message.contains("parse errors"))
                .count()
        };

        assert_eq!(parse_diagnostics(&clean), 0);
        assert!(parse_diagnostics(&slightly_broken) > 0);
        assert!(parse_diagnostics(&heavily_broken) > 0);
    }
    #[test]
    fn a_source_over_the_size_cap_is_reported_once() {
        use std::io::Write;

        let dir = test_dir().join("oversized");
        let _ = std::fs::remove_dir_all(&dir);
        std::fs::create_dir_all(&dir).unwrap();
        let path = dir.join("huge.py");

        let mut file = std::fs::File::create(&path).unwrap();
        let line = b"# filler line\n";
        let cap = 8 * 1024 * 1024;
        let mut written = 0usize;
        while written <= cap {
            file.write_all(line).unwrap();
            written += line.len();
        }
        drop(file);
        assert!(written > cap, "the file is over the cap");

        let result = extract_with_options(
            &[(path.clone(), LangId::Python)],
            &ExtractOptions::default(),
        );
        let extraction = &result.files[0];

        assert!(
            extraction.symbols.is_empty(),
            "an oversized file contributes no symbols"
        );
        let errors: Vec<&Diagnostic> = extraction
            .diagnostics
            .iter()
            .filter(|diagnostic| diagnostic.severity == Severity::Error)
            .collect();
        assert_eq!(
            errors.len(),
            1,
            "an oversized file reports exactly one error"
        );
        assert!(
            errors[0].message.contains("8388608"),
            "the error names the cap: {}",
            errors[0].message
        );

        let _ = std::fs::remove_dir_all(&dir);
    }

    #[test]
    fn two_passes_over_a_tree_agree() {
        let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/multi");
        let files = crate::input::discover_files(&root, None).unwrap();
        assert!(!files.is_empty(), "the fixture tree has files");

        let describe = |result: &ExtractionResult| -> Vec<String> {
            let mut described = Vec::new();
            for file in &result.files {
                described.push(format!("file {} {:?}", file.path.display(), file.lang));
                for symbol in &file.symbols {
                    described.push(format!(
                        "symbol {} {} {:?} {}",
                        symbol.id.to_raw(),
                        symbol.name,
                        symbol.kind,
                        symbol.source_range.byte_start
                    ));
                }
                for import in &file.imports {
                    described.push(format!(
                        "import {} {}",
                        import.import_specifier, import.range.byte_start
                    ));
                }
                for reference in &file.references {
                    described.push(format!(
                        "reference {} {}",
                        reference.name, reference.range.byte_start
                    ));
                }
            }
            described
        };

        let first = extract_with_options(&files, &ExtractOptions::default());
        let second = extract_with_options(&files, &ExtractOptions::default());

        assert_eq!(
            describe(&first),
            describe(&second),
            "two passes over the same tree produce the same extraction"
        );
    }
}