semtree_parse/
lib.rs

1//! **Tree-sitter parsing and chunk extraction for semtree.**
2//!
3//! Turns source files into structured [`Chunk`]s -
4//! functions, methods, structs, classes - aligned to real syntax boundaries
5//! instead of arbitrary line windows. Supports Rust, Python, JavaScript,
6//! TypeScript, TSX, Go, Java, C, C++, C#, Ruby, PHP, Kotlin, Scala, Swift,
7//! OCaml, Solidity, Lua, Zig and Emacs Lisp; non-code text falls back to
8//! fixed-size windows.
9//!
10//! Each language is a tree-sitter query in `src/lang/queries/`; adding one is a
11//! grammar dependency plus a `.scm` file, no per-language Rust.
12//!
13//! ```no_run
14//! use semtree_parse::extract_file;
15//!
16//! let chunks = extract_file(std::path::Path::new("src/lib.rs"))?;
17//! for c in &chunks {
18//!     println!("{:?} {:?}", c.kind, c.name);
19//! }
20//! # Ok::<(), semtree_parse::ParseError>(())
21//! ```
22
23mod error;
24mod lang;
25mod parser;
26mod text;
27
28pub use error::ParseError;
29pub use parser::{ParsedTree, SemtreeParser};
30pub use text::{chunk_text, is_text_file};
31
32use semtree_core::{Chunk, Language};
33
34pub fn parse_and_extract(source: &str, language: Language) -> Result<Vec<Chunk>, ParseError> {
35    let tree = SemtreeParser::parse(source, language)?;
36    Ok(lang::extract(&tree))
37}
38
39pub fn parse_and_extract_file(path: &std::path::Path) -> Result<Vec<Chunk>, ParseError> {
40    let tree = SemtreeParser::parse_file(path)?;
41    let mut chunks = lang::extract(&tree);
42    lang::shared::finalize_paths(&mut chunks, path);
43    Ok(chunks)
44}
45
46/// Extract chunks from any supported file - code or plain text.
47pub fn extract_file(path: &std::path::Path) -> Result<Vec<Chunk>, ParseError> {
48    if is_text_file(path) {
49        let source = std::fs::read_to_string(path)?;
50        return Ok(chunk_text(path, &source, 40, 5));
51    }
52    parse_and_extract_file(path)
53}
semtree_parse/lib.rs

semtree_parse/
lib.rs
