weavatrix_parse/
lib.rs

1//! Source tokenizer and structural extractor for repository intelligence.
2//!
3//! This crate exists because line-oriented scanning is wrong in ways that
4//! matter: a route written inside a comment becomes an endpoint, a `//` inside
5//! a string ends the line early, a declaration spanning three lines
6//! disappears, and a class body yields no methods. Every one of those is a
7//! tokenizer problem, so this crate starts with a tokenizer.
8//!
9//! It deliberately does not build an expression tree. Repository intelligence
10//! consumes declarations, imports, exports, calls and their spans - not
11//! operator precedence - so the structural pass walks the token stream and
12//! stops there. That keeps the crate small enough to own outright and fast
13//! enough to run over every file of a monorepo.
14//!
15//! No dependencies, no generated grammars, no C, `unsafe` forbidden.
16
17pub mod braced;
18mod contract_tokens;
19pub mod docs;
20pub mod facts;
21pub mod graphql;
22pub mod hcl;
23pub mod markup;
24pub mod protobuf;
25pub mod python;
26pub mod script;
27pub mod shell;
28pub mod sql;
29pub mod style;
30pub mod syntax;
31pub mod token;
32
33pub use facts::{
34    Contract, ContractKind, Declaration, DeclarationKind, Facts, GraphqlOperation, GraphqlType,
35    Import, ImportBinding, ParseDiagnostic, Reference, ReferenceKind, Span,
36};
37pub use syntax::{Language, Syntax};
38pub use token::{Mode, Token, TokenKind, Tokenizer, tokenize, tokenize_lite};
39
40/// Extracts structural facts from one source file.
41///
42/// Languages this crate can tokenize but has no structural model for yet -
43/// shell and YAML - return no facts rather than guesses.
44#[must_use]
45pub fn extract(source: &str, language: Language) -> Facts {
46    match language {
47        Language::JavaScript | Language::TypeScript => script::extract(source, language),
48        Language::Graphql => graphql::extract(source),
49        Language::Protobuf => protobuf::extract(source),
50        Language::Python => python::extract(source),
51        Language::Sql => sql::extract(source),
52        Language::Terraform => hcl::extract(source),
53        Language::Bash => shell::extract(source),
54        Language::Markdown | Language::Mdx | Language::ReStructuredText | Language::AsciiDoc => {
55            docs::extract(source, language)
56        }
57        Language::Html | Language::Xml => markup::extract(source, language),
58        Language::Css | Language::Scss => style::extract(source, language),
59        Language::Rust
60        | Language::Go
61        | Language::Java
62        | Language::CSharp
63        | Language::C
64        | Language::Cpp
65        | Language::Solidity
66        | Language::Swift => braced::extract(source, language),
67        _ => Facts::default(),
68    }
69}
70
71/// Extracts structural facts from a file, choosing the language by extension.
72#[must_use]
73pub fn extract_path(path: &str, source: &str) -> Option<Facts> {
74    let extension = path.rsplit_once('.')?.1;
75    let language = Language::from_extension(extension)?;
76    Some(extract(source, language))
77}
weavatrix_parse/lib.rs

weavatrix_parse/
lib.rs
